diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,10242 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 6793,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00014721036360959813,
+      "grad_norm": 29.718448026305357,
+      "learning_rate": 7.352941176470588e-10,
+      "logits/chosen": -3.0151314735412598,
+      "logits/rejected": -2.7100119590759277,
+      "logps/chosen": -416.9676208496094,
+      "logps/rejected": -87.89925384521484,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0014721036360959812,
+      "grad_norm": 34.522042629540266,
+      "learning_rate": 7.352941176470588e-09,
+      "logits/chosen": -2.7276546955108643,
+      "logits/rejected": -2.696035623550415,
+      "logps/chosen": -329.0189514160156,
+      "logps/rejected": -228.99288940429688,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.40740740299224854,
+      "rewards/chosen": 8.75515106599778e-05,
+      "rewards/margins": 0.0004485688405111432,
+      "rewards/rejected": -0.0003610174171626568,
+      "step": 10
+    },
+    {
+      "epoch": 0.0029442072721919624,
+      "grad_norm": 29.803944573152652,
+      "learning_rate": 1.4705882352941176e-08,
+      "logits/chosen": -2.7379086017608643,
+      "logits/rejected": -2.7686338424682617,
+      "logps/chosen": -257.52764892578125,
+      "logps/rejected": -231.6560516357422,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4333333969116211,
+      "rewards/chosen": -0.0014953905483707786,
+      "rewards/margins": -0.0011777242179960012,
+      "rewards/rejected": -0.00031766638858243823,
+      "step": 20
+    },
+    {
+      "epoch": 0.004416310908287944,
+      "grad_norm": 27.15125265339492,
+      "learning_rate": 2.2058823529411764e-08,
+      "logits/chosen": -2.8205478191375732,
+      "logits/rejected": -2.8088126182556152,
+      "logps/chosen": -202.71090698242188,
+      "logps/rejected": -208.66604614257812,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.36666664481163025,
+      "rewards/chosen": -0.000266118033323437,
+      "rewards/margins": -0.0014482138212770224,
+      "rewards/rejected": 0.0011820958461612463,
+      "step": 30
+    },
+    {
+      "epoch": 0.005888414544383925,
+      "grad_norm": 27.748767219344842,
+      "learning_rate": 2.941176470588235e-08,
+      "logits/chosen": -2.7897677421569824,
+      "logits/rejected": -2.7368621826171875,
+      "logps/chosen": -244.4735870361328,
+      "logps/rejected": -392.8205871582031,
+      "loss": 0.6939,
+      "rewards/accuracies": 0.3333333134651184,
+      "rewards/chosen": -0.00016161819803528488,
+      "rewards/margins": -0.0008207018254324794,
+      "rewards/rejected": 0.0006590837729163468,
+      "step": 40
+    },
+    {
+      "epoch": 0.007360518180479906,
+      "grad_norm": 38.32874910808029,
+      "learning_rate": 3.676470588235294e-08,
+      "logits/chosen": -2.756470203399658,
+      "logits/rejected": -2.7820448875427246,
+      "logps/chosen": -273.6413269042969,
+      "logps/rejected": -217.2002716064453,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.4333333373069763,
+      "rewards/chosen": 0.0007894287118688226,
+      "rewards/margins": 0.0006995893782004714,
+      "rewards/rejected": 8.983933366835117e-05,
+      "step": 50
+    },
+    {
+      "epoch": 0.008832621816575887,
+      "grad_norm": 34.64234916556576,
+      "learning_rate": 4.411764705882353e-08,
+      "logits/chosen": -2.8398597240448,
+      "logits/rejected": -2.82981014251709,
+      "logps/chosen": -320.47674560546875,
+      "logps/rejected": -330.9267883300781,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": 0.002479543210938573,
+      "rewards/margins": 0.001817501150071621,
+      "rewards/rejected": 0.0006620419444516301,
+      "step": 60
+    },
+    {
+      "epoch": 0.010304725452671868,
+      "grad_norm": 30.591214827639572,
+      "learning_rate": 5.147058823529411e-08,
+      "logits/chosen": -2.656278133392334,
+      "logits/rejected": -2.6301004886627197,
+      "logps/chosen": -247.33352661132812,
+      "logps/rejected": -297.91912841796875,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.5333333015441895,
+      "rewards/chosen": 0.0015019330894574523,
+      "rewards/margins": 0.0005758283077739179,
+      "rewards/rejected": 0.0009261048398911953,
+      "step": 70
+    },
+    {
+      "epoch": 0.01177682908876785,
+      "grad_norm": 29.311732036148747,
+      "learning_rate": 5.88235294117647e-08,
+      "logits/chosen": -2.886997938156128,
+      "logits/rejected": -2.8162360191345215,
+      "logps/chosen": -254.671630859375,
+      "logps/rejected": -228.0240020751953,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": 0.0005007624276913702,
+      "rewards/margins": -0.0004886452225036919,
+      "rewards/rejected": 0.0009894075337797403,
+      "step": 80
+    },
+    {
+      "epoch": 0.01324893272486383,
+      "grad_norm": 28.19375151099229,
+      "learning_rate": 6.617647058823529e-08,
+      "logits/chosen": -2.890990734100342,
+      "logits/rejected": -2.8566465377807617,
+      "logps/chosen": -297.2953186035156,
+      "logps/rejected": -230.1473388671875,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": 0.0029033743776381016,
+      "rewards/margins": 0.0014815034810453653,
+      "rewards/rejected": 0.001421870430931449,
+      "step": 90
+    },
+    {
+      "epoch": 0.014721036360959812,
+      "grad_norm": 29.517686848229378,
+      "learning_rate": 7.352941176470588e-08,
+      "logits/chosen": -2.8629086017608643,
+      "logits/rejected": -2.7863824367523193,
+      "logps/chosen": -256.60455322265625,
+      "logps/rejected": -211.39645385742188,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.43333330750465393,
+      "rewards/chosen": 0.0015266563277691603,
+      "rewards/margins": 0.0008079732651822269,
+      "rewards/rejected": 0.0007186831790022552,
+      "step": 100
+    },
+    {
+      "epoch": 0.016193139997055794,
+      "grad_norm": 28.80409325053622,
+      "learning_rate": 8.088235294117647e-08,
+      "logits/chosen": -2.901120901107788,
+      "logits/rejected": -2.8686041831970215,
+      "logps/chosen": -357.46038818359375,
+      "logps/rejected": -284.4308166503906,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.007152503821998835,
+      "rewards/margins": 0.002613933989778161,
+      "rewards/rejected": 0.00453856959939003,
+      "step": 110
+    },
+    {
+      "epoch": 0.017665243633151775,
+      "grad_norm": 36.06810060733407,
+      "learning_rate": 8.823529411764706e-08,
+      "logits/chosen": -2.766767740249634,
+      "logits/rejected": -2.7674272060394287,
+      "logps/chosen": -250.0283660888672,
+      "logps/rejected": -270.9175720214844,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.46666663885116577,
+      "rewards/chosen": 0.0038150239270180464,
+      "rewards/margins": 0.00018358106899540871,
+      "rewards/rejected": 0.003631443250924349,
+      "step": 120
+    },
+    {
+      "epoch": 0.019137347269247755,
+      "grad_norm": 32.75221872729468,
+      "learning_rate": 9.558823529411763e-08,
+      "logits/chosen": -2.8407626152038574,
+      "logits/rejected": -2.8098511695861816,
+      "logps/chosen": -284.9811096191406,
+      "logps/rejected": -217.84854125976562,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": 0.009724515490233898,
+      "rewards/margins": 0.006964170839637518,
+      "rewards/rejected": 0.0027603451162576675,
+      "step": 130
+    },
+    {
+      "epoch": 0.020609450905343735,
+      "grad_norm": 32.40114017346329,
+      "learning_rate": 1.0294117647058822e-07,
+      "logits/chosen": -2.87676739692688,
+      "logits/rejected": -2.8542282581329346,
+      "logps/chosen": -194.10653686523438,
+      "logps/rejected": -192.06259155273438,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": 0.011662951670587063,
+      "rewards/margins": 0.006347318179905415,
+      "rewards/rejected": 0.005315634422004223,
+      "step": 140
+    },
+    {
+      "epoch": 0.022081554541439716,
+      "grad_norm": 32.6183972925703,
+      "learning_rate": 1.1029411764705881e-07,
+      "logits/chosen": -2.836061954498291,
+      "logits/rejected": -2.7306556701660156,
+      "logps/chosen": -262.98590087890625,
+      "logps/rejected": -223.1548309326172,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 0.01592249795794487,
+      "rewards/margins": 0.0072413100861012936,
+      "rewards/rejected": 0.008681186474859715,
+      "step": 150
+    },
+    {
+      "epoch": 0.0235536581775357,
+      "grad_norm": 26.829953137326793,
+      "learning_rate": 1.176470588235294e-07,
+      "logits/chosen": -2.8711628913879395,
+      "logits/rejected": -2.7835397720336914,
+      "logps/chosen": -224.1538543701172,
+      "logps/rejected": -213.1654815673828,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": 0.014381324872374535,
+      "rewards/margins": 0.0020202070008963346,
+      "rewards/rejected": 0.012361117638647556,
+      "step": 160
+    },
+    {
+      "epoch": 0.02502576181363168,
+      "grad_norm": 43.31476786218589,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": -2.8009700775146484,
+      "logits/rejected": -2.8119311332702637,
+      "logps/chosen": -282.51092529296875,
+      "logps/rejected": -246.3946990966797,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": 0.03146267682313919,
+      "rewards/margins": 0.016998907551169395,
+      "rewards/rejected": 0.01446377020329237,
+      "step": 170
+    },
+    {
+      "epoch": 0.02649786544972766,
+      "grad_norm": 32.70338126838971,
+      "learning_rate": 1.3235294117647057e-07,
+      "logits/chosen": -2.873365640640259,
+      "logits/rejected": -2.8650002479553223,
+      "logps/chosen": -198.95913696289062,
+      "logps/rejected": -216.2865447998047,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": 0.029899191111326218,
+      "rewards/margins": 0.012989306822419167,
+      "rewards/rejected": 0.016909882426261902,
+      "step": 180
+    },
+    {
+      "epoch": 0.02796996908582364,
+      "grad_norm": 27.647809254073966,
+      "learning_rate": 1.3970588235294117e-07,
+      "logits/chosen": -2.8443760871887207,
+      "logits/rejected": -2.8531670570373535,
+      "logps/chosen": -238.43276977539062,
+      "logps/rejected": -254.6396026611328,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": 0.03018432855606079,
+      "rewards/margins": 0.011908235028386116,
+      "rewards/rejected": 0.018276093527674675,
+      "step": 190
+    },
+    {
+      "epoch": 0.029442072721919624,
+      "grad_norm": 31.006923353950068,
+      "learning_rate": 1.4705882352941175e-07,
+      "logits/chosen": -2.8793869018554688,
+      "logits/rejected": -2.809866428375244,
+      "logps/chosen": -319.7313537597656,
+      "logps/rejected": -248.39608764648438,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.04280875250697136,
+      "rewards/margins": 0.02830171212553978,
+      "rewards/rejected": 0.014507037587463856,
+      "step": 200
+    },
+    {
+      "epoch": 0.030914176358015605,
+      "grad_norm": 35.15155577786183,
+      "learning_rate": 1.5441176470588236e-07,
+      "logits/chosen": -2.7579071521759033,
+      "logits/rejected": -2.750349760055542,
+      "logps/chosen": -250.6804962158203,
+      "logps/rejected": -204.61766052246094,
+      "loss": 0.676,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": 0.043508581817150116,
+      "rewards/margins": 0.02954995632171631,
+      "rewards/rejected": 0.013958625495433807,
+      "step": 210
+    },
+    {
+      "epoch": 0.03238627999411159,
+      "grad_norm": 41.541119045331676,
+      "learning_rate": 1.6176470588235293e-07,
+      "logits/chosen": -2.755091667175293,
+      "logits/rejected": -2.730134963989258,
+      "logps/chosen": -241.91476440429688,
+      "logps/rejected": -188.8052520751953,
+      "loss": 0.6712,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.041111722588539124,
+      "rewards/margins": 0.034332290291786194,
+      "rewards/rejected": 0.006779427640140057,
+      "step": 220
+    },
+    {
+      "epoch": 0.033858383630207566,
+      "grad_norm": 35.780007512350444,
+      "learning_rate": 1.6911764705882354e-07,
+      "logits/chosen": -2.8124828338623047,
+      "logits/rejected": -2.7572226524353027,
+      "logps/chosen": -346.2286682128906,
+      "logps/rejected": -255.1610107421875,
+      "loss": 0.6761,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": 0.0636453926563263,
+      "rewards/margins": 0.04507563263177872,
+      "rewards/rejected": 0.018569760024547577,
+      "step": 230
+    },
+    {
+      "epoch": 0.03533048726630355,
+      "grad_norm": 24.867225994165324,
+      "learning_rate": 1.764705882352941e-07,
+      "logits/chosen": -2.8328232765197754,
+      "logits/rejected": -2.764782428741455,
+      "logps/chosen": -259.3244934082031,
+      "logps/rejected": -253.60427856445312,
+      "loss": 0.675,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03747386485338211,
+      "rewards/margins": 0.03954165801405907,
+      "rewards/rejected": -0.0020677968859672546,
+      "step": 240
+    },
+    {
+      "epoch": 0.036802590902399526,
+      "grad_norm": 31.837234874996586,
+      "learning_rate": 1.8382352941176472e-07,
+      "logits/chosen": -2.6840827465057373,
+      "logits/rejected": -2.723935842514038,
+      "logps/chosen": -238.89794921875,
+      "logps/rejected": -240.5414581298828,
+      "loss": 0.6754,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.032973356544971466,
+      "rewards/margins": 0.048632390797138214,
+      "rewards/rejected": -0.015659034252166748,
+      "step": 250
+    },
+    {
+      "epoch": 0.03827469453849551,
+      "grad_norm": 29.646802079354163,
+      "learning_rate": 1.9117647058823527e-07,
+      "logits/chosen": -2.88181734085083,
+      "logits/rejected": -2.779343605041504,
+      "logps/chosen": -266.6003723144531,
+      "logps/rejected": -222.57400512695312,
+      "loss": 0.673,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.003907797392457724,
+      "rewards/margins": 0.012058199383318424,
+      "rewards/rejected": -0.01596599444746971,
+      "step": 260
+    },
+    {
+      "epoch": 0.039746798174591494,
+      "grad_norm": 26.851190889186874,
+      "learning_rate": 1.9852941176470587e-07,
+      "logits/chosen": -2.799879789352417,
+      "logits/rejected": -2.848215341567993,
+      "logps/chosen": -292.47210693359375,
+      "logps/rejected": -280.51190185546875,
+      "loss": 0.6589,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.027005229145288467,
+      "rewards/margins": 0.039683397859334946,
+      "rewards/rejected": -0.012678168714046478,
+      "step": 270
+    },
+    {
+      "epoch": 0.04121890181068747,
+      "grad_norm": 30.64659123108073,
+      "learning_rate": 2.0588235294117645e-07,
+      "logits/chosen": -2.840538263320923,
+      "logits/rejected": -2.781792640686035,
+      "logps/chosen": -315.31658935546875,
+      "logps/rejected": -288.5542297363281,
+      "loss": 0.6538,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.05827077478170395,
+      "rewards/margins": 0.07762565463781357,
+      "rewards/rejected": -0.01935487426817417,
+      "step": 280
+    },
+    {
+      "epoch": 0.042691005446783455,
+      "grad_norm": 38.622609391819914,
+      "learning_rate": 2.1323529411764705e-07,
+      "logits/chosen": -2.829793930053711,
+      "logits/rejected": -2.8271484375,
+      "logps/chosen": -268.45330810546875,
+      "logps/rejected": -226.24526977539062,
+      "loss": 0.6623,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": 0.04187547042965889,
+      "rewards/margins": 0.09543410688638687,
+      "rewards/rejected": -0.05355863645672798,
+      "step": 290
+    },
+    {
+      "epoch": 0.04416310908287943,
+      "grad_norm": 32.02994765741851,
+      "learning_rate": 2.2058823529411763e-07,
+      "logits/chosen": -2.820500373840332,
+      "logits/rejected": -2.7968363761901855,
+      "logps/chosen": -271.98992919921875,
+      "logps/rejected": -247.7967987060547,
+      "loss": 0.658,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": -0.03903895989060402,
+      "rewards/margins": 0.0660947933793068,
+      "rewards/rejected": -0.10513375699520111,
+      "step": 300
+    },
+    {
+      "epoch": 0.045635212718975415,
+      "grad_norm": 45.6308084481084,
+      "learning_rate": 2.2794117647058823e-07,
+      "logits/chosen": -2.9065310955047607,
+      "logits/rejected": -2.8418185710906982,
+      "logps/chosen": -309.17889404296875,
+      "logps/rejected": -269.48394775390625,
+      "loss": 0.6641,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.003005195641890168,
+      "rewards/margins": 0.10960595309734344,
+      "rewards/rejected": -0.10660076141357422,
+      "step": 310
+    },
+    {
+      "epoch": 0.0471073163550714,
+      "grad_norm": 38.680300557067135,
+      "learning_rate": 2.352941176470588e-07,
+      "logits/chosen": -2.7667911052703857,
+      "logits/rejected": -2.745961904525757,
+      "logps/chosen": -259.071044921875,
+      "logps/rejected": -268.68756103515625,
+      "loss": 0.6419,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.04463445395231247,
+      "rewards/margins": 0.1286865472793579,
+      "rewards/rejected": -0.1733209788799286,
+      "step": 320
+    },
+    {
+      "epoch": 0.048579419991167376,
+      "grad_norm": 43.0929608927417,
+      "learning_rate": 2.426470588235294e-07,
+      "logits/chosen": -2.805534839630127,
+      "logits/rejected": -2.7685582637786865,
+      "logps/chosen": -234.90036010742188,
+      "logps/rejected": -213.7690887451172,
+      "loss": 0.6416,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -0.04509614408016205,
+      "rewards/margins": 0.13314001262187958,
+      "rewards/rejected": -0.17823615670204163,
+      "step": 330
+    },
+    {
+      "epoch": 0.05005152362726336,
+      "grad_norm": 32.26871001944957,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -2.8668112754821777,
+      "logits/rejected": -2.820902109146118,
+      "logps/chosen": -304.05535888671875,
+      "logps/rejected": -353.029296875,
+      "loss": 0.6517,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": -0.08451329171657562,
+      "rewards/margins": 0.08599463105201721,
+      "rewards/rejected": -0.17050793766975403,
+      "step": 340
+    },
+    {
+      "epoch": 0.051523627263359344,
+      "grad_norm": 44.69532980871668,
+      "learning_rate": 2.5735294117647057e-07,
+      "logits/chosen": -2.7562334537506104,
+      "logits/rejected": -2.730844020843506,
+      "logps/chosen": -316.5646057128906,
+      "logps/rejected": -269.04498291015625,
+      "loss": 0.6456,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.11849607527256012,
+      "rewards/margins": 0.12490792572498322,
+      "rewards/rejected": -0.24340395629405975,
+      "step": 350
+    },
+    {
+      "epoch": 0.05299573089945532,
+      "grad_norm": 34.892260900939654,
+      "learning_rate": 2.6470588235294114e-07,
+      "logits/chosen": -2.8204593658447266,
+      "logits/rejected": -2.8178982734680176,
+      "logps/chosen": -250.16171264648438,
+      "logps/rejected": -244.96011352539062,
+      "loss": 0.6529,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.14288079738616943,
+      "rewards/margins": 0.11003166437149048,
+      "rewards/rejected": -0.2529124915599823,
+      "step": 360
+    },
+    {
+      "epoch": 0.054467834535551304,
+      "grad_norm": 40.19716854439912,
+      "learning_rate": 2.720588235294117e-07,
+      "logits/chosen": -2.8387558460235596,
+      "logits/rejected": -2.8843367099761963,
+      "logps/chosen": -269.55828857421875,
+      "logps/rejected": -285.814453125,
+      "loss": 0.597,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -0.0735350102186203,
+      "rewards/margins": 0.20208704471588135,
+      "rewards/rejected": -0.27562204003334045,
+      "step": 370
+    },
+    {
+      "epoch": 0.05593993817164728,
+      "grad_norm": 40.430296299126645,
+      "learning_rate": 2.7941176470588235e-07,
+      "logits/chosen": -2.9129586219787598,
+      "logits/rejected": -2.863222599029541,
+      "logps/chosen": -300.028076171875,
+      "logps/rejected": -262.0947265625,
+      "loss": 0.6348,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3237457573413849,
+      "rewards/margins": 0.13909229636192322,
+      "rewards/rejected": -0.4628380835056305,
+      "step": 380
+    },
+    {
+      "epoch": 0.057412041807743265,
+      "grad_norm": 38.1179702652076,
+      "learning_rate": 2.8676470588235293e-07,
+      "logits/chosen": -2.856393337249756,
+      "logits/rejected": -2.845912456512451,
+      "logps/chosen": -304.79083251953125,
+      "logps/rejected": -298.27838134765625,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": -0.44157400727272034,
+      "rewards/margins": -0.01729520596563816,
+      "rewards/rejected": -0.42427879571914673,
+      "step": 390
+    },
+    {
+      "epoch": 0.05888414544383925,
+      "grad_norm": 37.66484657174034,
+      "learning_rate": 2.941176470588235e-07,
+      "logits/chosen": -2.846362590789795,
+      "logits/rejected": -2.7886412143707275,
+      "logps/chosen": -251.76260375976562,
+      "logps/rejected": -277.6621398925781,
+      "loss": 0.6068,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.39646148681640625,
+      "rewards/margins": 0.20760174095630646,
+      "rewards/rejected": -0.6040632128715515,
+      "step": 400
+    },
+    {
+      "epoch": 0.060356249079935226,
+      "grad_norm": 44.823295544217466,
+      "learning_rate": 3.014705882352941e-07,
+      "logits/chosen": -2.7775685787200928,
+      "logits/rejected": -2.7247066497802734,
+      "logps/chosen": -342.48760986328125,
+      "logps/rejected": -346.5843200683594,
+      "loss": 0.6163,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.36908066272735596,
+      "rewards/margins": 0.1995377540588379,
+      "rewards/rejected": -0.5686183571815491,
+      "step": 410
+    },
+    {
+      "epoch": 0.06182835271603121,
+      "grad_norm": 57.91546494547815,
+      "learning_rate": 3.088235294117647e-07,
+      "logits/chosen": -2.9392282962799072,
+      "logits/rejected": -2.874932050704956,
+      "logps/chosen": -355.48284912109375,
+      "logps/rejected": -334.55877685546875,
+      "loss": 0.6611,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -0.35706859827041626,
+      "rewards/margins": 0.08930377662181854,
+      "rewards/rejected": -0.4463724195957184,
+      "step": 420
+    },
+    {
+      "epoch": 0.0633004563521272,
+      "grad_norm": 40.864948374445554,
+      "learning_rate": 3.161764705882353e-07,
+      "logits/chosen": -2.813310146331787,
+      "logits/rejected": -2.8060102462768555,
+      "logps/chosen": -189.114013671875,
+      "logps/rejected": -223.2728729248047,
+      "loss": 0.6161,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1683691293001175,
+      "rewards/margins": 0.22235913574695587,
+      "rewards/rejected": -0.39072829484939575,
+      "step": 430
+    },
+    {
+      "epoch": 0.06477255998822318,
+      "grad_norm": 39.57400537972088,
+      "learning_rate": 3.2352941176470586e-07,
+      "logits/chosen": -2.841413974761963,
+      "logits/rejected": -2.83341646194458,
+      "logps/chosen": -226.3544921875,
+      "logps/rejected": -234.64328002929688,
+      "loss": 0.595,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -0.12442722171545029,
+      "rewards/margins": 0.3463088274002075,
+      "rewards/rejected": -0.470736026763916,
+      "step": 440
+    },
+    {
+      "epoch": 0.06624466362431915,
+      "grad_norm": 40.15876245255608,
+      "learning_rate": 3.3088235294117644e-07,
+      "logits/chosen": -2.8721110820770264,
+      "logits/rejected": -2.8159689903259277,
+      "logps/chosen": -383.03045654296875,
+      "logps/rejected": -354.8152160644531,
+      "loss": 0.617,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": -0.5107864141464233,
+      "rewards/margins": 0.07485867291688919,
+      "rewards/rejected": -0.5856450796127319,
+      "step": 450
+    },
+    {
+      "epoch": 0.06771676726041513,
+      "grad_norm": 50.420023450070595,
+      "learning_rate": 3.3823529411764707e-07,
+      "logits/chosen": -2.9715359210968018,
+      "logits/rejected": -2.8942055702209473,
+      "logps/chosen": -316.4211730957031,
+      "logps/rejected": -294.3646240234375,
+      "loss": 0.6338,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.3568728566169739,
+      "rewards/margins": 0.4465089738368988,
+      "rewards/rejected": -0.8033817410469055,
+      "step": 460
+    },
+    {
+      "epoch": 0.06918887089651111,
+      "grad_norm": 45.45079803725552,
+      "learning_rate": 3.4558823529411765e-07,
+      "logits/chosen": -2.894871950149536,
+      "logits/rejected": -2.882598638534546,
+      "logps/chosen": -268.9466247558594,
+      "logps/rejected": -284.09405517578125,
+      "loss": 0.5799,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -0.5613805651664734,
+      "rewards/margins": 0.30428066849708557,
+      "rewards/rejected": -0.8656612634658813,
+      "step": 470
+    },
+    {
+      "epoch": 0.0706609745326071,
+      "grad_norm": 39.98380606716504,
+      "learning_rate": 3.529411764705882e-07,
+      "logits/chosen": -2.808845281600952,
+      "logits/rejected": -2.765450954437256,
+      "logps/chosen": -341.7030334472656,
+      "logps/rejected": -362.5021667480469,
+      "loss": 0.5428,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -0.6110513806343079,
+      "rewards/margins": 0.6107190847396851,
+      "rewards/rejected": -1.2217704057693481,
+      "step": 480
+    },
+    {
+      "epoch": 0.07213307816870308,
+      "grad_norm": 52.16434389424912,
+      "learning_rate": 3.602941176470588e-07,
+      "logits/chosen": -2.913093090057373,
+      "logits/rejected": -2.8623852729797363,
+      "logps/chosen": -376.265625,
+      "logps/rejected": -357.77996826171875,
+      "loss": 0.6358,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.6043509244918823,
+      "rewards/margins": 0.2012513428926468,
+      "rewards/rejected": -0.8056022524833679,
+      "step": 490
+    },
+    {
+      "epoch": 0.07360518180479905,
+      "grad_norm": 49.42423787089578,
+      "learning_rate": 3.6764705882352943e-07,
+      "logits/chosen": -2.8766541481018066,
+      "logits/rejected": -2.901610851287842,
+      "logps/chosen": -248.31686401367188,
+      "logps/rejected": -313.4378967285156,
+      "loss": 0.5365,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -0.4482509195804596,
+      "rewards/margins": 0.4299069344997406,
+      "rewards/rejected": -0.8781577944755554,
+      "step": 500
+    },
+    {
+      "epoch": 0.07507728544089504,
+      "grad_norm": 32.57183193077679,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": -2.821455717086792,
+      "logits/rejected": -2.8248238563537598,
+      "logps/chosen": -348.87811279296875,
+      "logps/rejected": -407.82110595703125,
+      "loss": 0.5858,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -0.3897936940193176,
+      "rewards/margins": 0.7289079427719116,
+      "rewards/rejected": -1.1187015771865845,
+      "step": 510
+    },
+    {
+      "epoch": 0.07654938907699102,
+      "grad_norm": 70.79198965743817,
+      "learning_rate": 3.8235294117647053e-07,
+      "logits/chosen": -2.70387601852417,
+      "logits/rejected": -2.607253074645996,
+      "logps/chosen": -290.9710388183594,
+      "logps/rejected": -309.3666687011719,
+      "loss": 0.6392,
+      "rewards/accuracies": 0.6333333849906921,
+      "rewards/chosen": -0.7217598557472229,
+      "rewards/margins": 0.16696974635124207,
+      "rewards/rejected": -0.8887295722961426,
+      "step": 520
+    },
+    {
+      "epoch": 0.078021492713087,
+      "grad_norm": 68.7838639999348,
+      "learning_rate": 3.8970588235294116e-07,
+      "logits/chosen": -2.878333330154419,
+      "logits/rejected": -2.8191580772399902,
+      "logps/chosen": -337.52203369140625,
+      "logps/rejected": -348.9778747558594,
+      "loss": 0.6487,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -0.6876872181892395,
+      "rewards/margins": 0.25353384017944336,
+      "rewards/rejected": -0.9412209391593933,
+      "step": 530
+    },
+    {
+      "epoch": 0.07949359634918299,
+      "grad_norm": 72.45119032304852,
+      "learning_rate": 3.9705882352941174e-07,
+      "logits/chosen": -3.028770923614502,
+      "logits/rejected": -2.9213104248046875,
+      "logps/chosen": -358.0929260253906,
+      "logps/rejected": -314.4250183105469,
+      "loss": 0.6274,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": -0.4906277060508728,
+      "rewards/margins": 0.09770762920379639,
+      "rewards/rejected": -0.588335394859314,
+      "step": 540
+    },
+    {
+      "epoch": 0.08096569998527896,
+      "grad_norm": 42.155038860783506,
+      "learning_rate": 4.044117647058823e-07,
+      "logits/chosen": -2.8411173820495605,
+      "logits/rejected": -2.8484601974487305,
+      "logps/chosen": -366.4306335449219,
+      "logps/rejected": -394.5599060058594,
+      "loss": 0.509,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -0.3539455533027649,
+      "rewards/margins": 0.6814759373664856,
+      "rewards/rejected": -1.0354214906692505,
+      "step": 550
+    },
+    {
+      "epoch": 0.08243780362137494,
+      "grad_norm": 59.00223055195894,
+      "learning_rate": 4.117647058823529e-07,
+      "logits/chosen": -2.9224987030029297,
+      "logits/rejected": -2.854747772216797,
+      "logps/chosen": -316.6570739746094,
+      "logps/rejected": -348.1865234375,
+      "loss": 0.5664,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.7669118642807007,
+      "rewards/margins": 0.4014472961425781,
+      "rewards/rejected": -1.1683591604232788,
+      "step": 560
+    },
+    {
+      "epoch": 0.08390990725747093,
+      "grad_norm": 158.17054517954566,
+      "learning_rate": 4.191176470588235e-07,
+      "logits/chosen": -2.860236883163452,
+      "logits/rejected": -2.782071590423584,
+      "logps/chosen": -366.7201232910156,
+      "logps/rejected": -369.0834655761719,
+      "loss": 0.6191,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.7171522974967957,
+      "rewards/margins": 0.708254873752594,
+      "rewards/rejected": -1.4254071712493896,
+      "step": 570
+    },
+    {
+      "epoch": 0.08538201089356691,
+      "grad_norm": 100.00875893504237,
+      "learning_rate": 4.264705882352941e-07,
+      "logits/chosen": -2.7357230186462402,
+      "logits/rejected": -2.732067584991455,
+      "logps/chosen": -403.13421630859375,
+      "logps/rejected": -354.78961181640625,
+      "loss": 0.5898,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.0905609130859375,
+      "rewards/margins": 0.23605385422706604,
+      "rewards/rejected": -1.3266146183013916,
+      "step": 580
+    },
+    {
+      "epoch": 0.08685411452966289,
+      "grad_norm": 105.63343146580014,
+      "learning_rate": 4.338235294117647e-07,
+      "logits/chosen": -2.7384467124938965,
+      "logits/rejected": -2.7308027744293213,
+      "logps/chosen": -354.20928955078125,
+      "logps/rejected": -388.06048583984375,
+      "loss": 0.5831,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -0.7502145171165466,
+      "rewards/margins": 0.5446814298629761,
+      "rewards/rejected": -1.294895887374878,
+      "step": 590
+    },
+    {
+      "epoch": 0.08832621816575886,
+      "grad_norm": 114.38595516000808,
+      "learning_rate": 4.4117647058823526e-07,
+      "logits/chosen": -2.5501580238342285,
+      "logits/rejected": -2.652919054031372,
+      "logps/chosen": -336.08380126953125,
+      "logps/rejected": -418.5784606933594,
+      "loss": 0.5505,
+      "rewards/accuracies": 0.73333340883255,
+      "rewards/chosen": -0.7092531323432922,
+      "rewards/margins": 0.719337522983551,
+      "rewards/rejected": -1.4285906553268433,
+      "step": 600
+    },
+    {
+      "epoch": 0.08979832180185485,
+      "grad_norm": 141.59819490328758,
+      "learning_rate": 4.485294117647059e-07,
+      "logits/chosen": -2.716247081756592,
+      "logits/rejected": -2.6971218585968018,
+      "logps/chosen": -330.3377990722656,
+      "logps/rejected": -390.364013671875,
+      "loss": 0.5446,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.7154955863952637,
+      "rewards/margins": 0.6936029195785522,
+      "rewards/rejected": -1.4090986251831055,
+      "step": 610
+    },
+    {
+      "epoch": 0.09127042543795083,
+      "grad_norm": 46.65751534451082,
+      "learning_rate": 4.5588235294117646e-07,
+      "logits/chosen": -2.858609914779663,
+      "logits/rejected": -2.794933557510376,
+      "logps/chosen": -318.4283752441406,
+      "logps/rejected": -318.87432861328125,
+      "loss": 0.5765,
+      "rewards/accuracies": 0.5999999642372131,
+      "rewards/chosen": -0.773520290851593,
+      "rewards/margins": 0.44950708746910095,
+      "rewards/rejected": -1.2230274677276611,
+      "step": 620
+    },
+    {
+      "epoch": 0.09274252907404681,
+      "grad_norm": 63.97315843764817,
+      "learning_rate": 4.6323529411764704e-07,
+      "logits/chosen": -2.817518711090088,
+      "logits/rejected": -2.7873594760894775,
+      "logps/chosen": -395.9840087890625,
+      "logps/rejected": -406.97479248046875,
+      "loss": 0.4769,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.8255616426467896,
+      "rewards/margins": 0.7154530882835388,
+      "rewards/rejected": -1.5410146713256836,
+      "step": 630
+    },
+    {
+      "epoch": 0.0942146327101428,
+      "grad_norm": 281.24021060780353,
+      "learning_rate": 4.705882352941176e-07,
+      "logits/chosen": -2.747581958770752,
+      "logits/rejected": -2.699913501739502,
+      "logps/chosen": -392.9543762207031,
+      "logps/rejected": -419.4530334472656,
+      "loss": 0.6503,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -1.1352064609527588,
+      "rewards/margins": 0.42551788687705994,
+      "rewards/rejected": -1.5607244968414307,
+      "step": 640
+    },
+    {
+      "epoch": 0.09568673634623878,
+      "grad_norm": 60.53577553719157,
+      "learning_rate": 4.779411764705882e-07,
+      "logits/chosen": -2.7471470832824707,
+      "logits/rejected": -2.6654038429260254,
+      "logps/chosen": -347.45501708984375,
+      "logps/rejected": -396.5620422363281,
+      "loss": 0.5248,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -0.5082324147224426,
+      "rewards/margins": 0.6192538738250732,
+      "rewards/rejected": -1.1274863481521606,
+      "step": 650
+    },
+    {
+      "epoch": 0.09715883998233475,
+      "grad_norm": 82.56737879385132,
+      "learning_rate": 4.852941176470588e-07,
+      "logits/chosen": -2.751802444458008,
+      "logits/rejected": -2.6738271713256836,
+      "logps/chosen": -337.10150146484375,
+      "logps/rejected": -350.31634521484375,
+      "loss": 0.575,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.8105621337890625,
+      "rewards/margins": 0.44342517852783203,
+      "rewards/rejected": -1.2539875507354736,
+      "step": 660
+    },
+    {
+      "epoch": 0.09863094361843074,
+      "grad_norm": 126.84675528859458,
+      "learning_rate": 4.926470588235295e-07,
+      "logits/chosen": -2.6607487201690674,
+      "logits/rejected": -2.690328359603882,
+      "logps/chosen": -375.0699157714844,
+      "logps/rejected": -356.2629699707031,
+      "loss": 0.5818,
+      "rewards/accuracies": 0.5666667222976685,
+      "rewards/chosen": -0.9144572019577026,
+      "rewards/margins": 0.3163110613822937,
+      "rewards/rejected": -1.2307683229446411,
+      "step": 670
+    },
+    {
+      "epoch": 0.10010304725452672,
+      "grad_norm": 88.01654938748439,
+      "learning_rate": 5e-07,
+      "logits/chosen": -2.810364246368408,
+      "logits/rejected": -2.6994426250457764,
+      "logps/chosen": -327.703369140625,
+      "logps/rejected": -312.79620361328125,
+      "loss": 0.6337,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.830489993095398,
+      "rewards/margins": 0.4034651815891266,
+      "rewards/rejected": -1.2339551448822021,
+      "step": 680
+    },
+    {
+      "epoch": 0.1015751508906227,
+      "grad_norm": 67.9023375850433,
+      "learning_rate": 4.999966985858302e-07,
+      "logits/chosen": -2.7782535552978516,
+      "logits/rejected": -2.7502403259277344,
+      "logps/chosen": -320.7219543457031,
+      "logps/rejected": -359.96759033203125,
+      "loss": 0.5899,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.7388945817947388,
+      "rewards/margins": 0.2782880365848541,
+      "rewards/rejected": -1.0171825885772705,
+      "step": 690
+    },
+    {
+      "epoch": 0.10304725452671869,
+      "grad_norm": 63.88394385411192,
+      "learning_rate": 4.999867944305156e-07,
+      "logits/chosen": -2.7173736095428467,
+      "logits/rejected": -2.7157700061798096,
+      "logps/chosen": -260.1662902832031,
+      "logps/rejected": -324.9969787597656,
+      "loss": 0.5417,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.646327793598175,
+      "rewards/margins": 0.5964962244033813,
+      "rewards/rejected": -1.2428240776062012,
+      "step": 700
+    },
+    {
+      "epoch": 0.10451935816281466,
+      "grad_norm": 82.12543215083453,
+      "learning_rate": 4.99970287795638e-07,
+      "logits/chosen": -2.7450194358825684,
+      "logits/rejected": -2.6951887607574463,
+      "logps/chosen": -377.6488037109375,
+      "logps/rejected": -471.7896423339844,
+      "loss": 0.6345,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -1.2469216585159302,
+      "rewards/margins": 0.43245401978492737,
+      "rewards/rejected": -1.6793756484985352,
+      "step": 710
+    },
+    {
+      "epoch": 0.10599146179891064,
+      "grad_norm": 76.26325826475778,
+      "learning_rate": 4.999471791171592e-07,
+      "logits/chosen": -2.6871371269226074,
+      "logits/rejected": -2.713042736053467,
+      "logps/chosen": -373.04498291015625,
+      "logps/rejected": -366.93609619140625,
+      "loss": 0.5365,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.154412031173706,
+      "rewards/margins": 0.4296625256538391,
+      "rewards/rejected": -1.58407461643219,
+      "step": 720
+    },
+    {
+      "epoch": 0.10746356543500662,
+      "grad_norm": 72.59515441979644,
+      "learning_rate": 4.999174690054098e-07,
+      "logits/chosen": -2.7032949924468994,
+      "logits/rejected": -2.706186532974243,
+      "logps/chosen": -378.016357421875,
+      "logps/rejected": -493.35150146484375,
+      "loss": 0.6098,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.5206677913665771,
+      "rewards/margins": 0.39631330966949463,
+      "rewards/rejected": -1.9169813394546509,
+      "step": 730
+    },
+    {
+      "epoch": 0.10893566907110261,
+      "grad_norm": 77.08798499260584,
+      "learning_rate": 4.998811582450728e-07,
+      "logits/chosen": -2.7526307106018066,
+      "logits/rejected": -2.770782947540283,
+      "logps/chosen": -417.216064453125,
+      "logps/rejected": -473.040771484375,
+      "loss": 0.5572,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.1277623176574707,
+      "rewards/margins": 0.6581010818481445,
+      "rewards/rejected": -1.7858635187149048,
+      "step": 740
+    },
+    {
+      "epoch": 0.11040777270719859,
+      "grad_norm": 92.20853458400707,
+      "learning_rate": 4.998382477951632e-07,
+      "logits/chosen": -2.7655673027038574,
+      "logits/rejected": -2.77188777923584,
+      "logps/chosen": -330.7748107910156,
+      "logps/rejected": -394.17230224609375,
+      "loss": 0.6361,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.120185375213623,
+      "rewards/margins": 0.358142226934433,
+      "rewards/rejected": -1.4783275127410889,
+      "step": 750
+    },
+    {
+      "epoch": 0.11187987634329456,
+      "grad_norm": 64.87979385908508,
+      "learning_rate": 4.997887387890022e-07,
+      "logits/chosen": -2.694700241088867,
+      "logits/rejected": -2.7389893531799316,
+      "logps/chosen": -423.99481201171875,
+      "logps/rejected": -472.5301818847656,
+      "loss": 0.4996,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -0.7210303544998169,
+      "rewards/margins": 0.6826065182685852,
+      "rewards/rejected": -1.4036369323730469,
+      "step": 760
+    },
+    {
+      "epoch": 0.11335197997939055,
+      "grad_norm": 62.14673998391493,
+      "learning_rate": 4.997326325341876e-07,
+      "logits/chosen": -2.865445852279663,
+      "logits/rejected": -2.785827875137329,
+      "logps/chosen": -351.143798828125,
+      "logps/rejected": -376.88116455078125,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.141156554222107,
+      "rewards/margins": 0.18905548751354218,
+      "rewards/rejected": -1.3302119970321655,
+      "step": 770
+    },
+    {
+      "epoch": 0.11482408361548653,
+      "grad_norm": 50.25048989730604,
+      "learning_rate": 4.996699305125597e-07,
+      "logits/chosen": -2.6305692195892334,
+      "logits/rejected": -2.6139140129089355,
+      "logps/chosen": -318.23907470703125,
+      "logps/rejected": -363.19854736328125,
+      "loss": 0.6313,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -0.9152249097824097,
+      "rewards/margins": 0.6227025389671326,
+      "rewards/rejected": -1.537927508354187,
+      "step": 780
+    },
+    {
+      "epoch": 0.11629618725158251,
+      "grad_norm": 60.098691114202936,
+      "learning_rate": 4.996006343801608e-07,
+      "logits/chosen": -2.580573320388794,
+      "logits/rejected": -2.538261651992798,
+      "logps/chosen": -304.2982482910156,
+      "logps/rejected": -375.99017333984375,
+      "loss": 0.5741,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -0.7302695512771606,
+      "rewards/margins": 0.7375911474227905,
+      "rewards/rejected": -1.4678608179092407,
+      "step": 790
+    },
+    {
+      "epoch": 0.1177682908876785,
+      "grad_norm": 94.19572824034412,
+      "learning_rate": 4.99524745967193e-07,
+      "logits/chosen": -2.521172046661377,
+      "logits/rejected": -2.49949312210083,
+      "logps/chosen": -468.8832092285156,
+      "logps/rejected": -470.048583984375,
+      "loss": 0.5629,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1481328010559082,
+      "rewards/margins": 0.7506190538406372,
+      "rewards/rejected": -1.8987518548965454,
+      "step": 800
+    },
+    {
+      "epoch": 0.11924039452377447,
+      "grad_norm": 77.02767987142049,
+      "learning_rate": 4.994422672779687e-07,
+      "logits/chosen": -2.4656782150268555,
+      "logits/rejected": -2.4513707160949707,
+      "logps/chosen": -349.1130065917969,
+      "logps/rejected": -448.8529357910156,
+      "loss": 0.5245,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.2570167779922485,
+      "rewards/margins": 0.6979629993438721,
+      "rewards/rejected": -1.954979658126831,
+      "step": 810
+    },
+    {
+      "epoch": 0.12071249815987045,
+      "grad_norm": 107.62300360051363,
+      "learning_rate": 4.993532004908588e-07,
+      "logits/chosen": -2.4160711765289307,
+      "logits/rejected": -2.288529872894287,
+      "logps/chosen": -371.8260803222656,
+      "logps/rejected": -403.8309020996094,
+      "loss": 0.604,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.372617244720459,
+      "rewards/margins": 0.3440636694431305,
+      "rewards/rejected": -1.716680884361267,
+      "step": 820
+    },
+    {
+      "epoch": 0.12218460179596644,
+      "grad_norm": 90.2537296603263,
+      "learning_rate": 4.992575479582337e-07,
+      "logits/chosen": -2.3595759868621826,
+      "logits/rejected": -2.309431791305542,
+      "logps/chosen": -489.00909423828125,
+      "logps/rejected": -533.1735229492188,
+      "loss": 0.5051,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.147862195968628,
+      "rewards/margins": 0.6809053421020508,
+      "rewards/rejected": -1.8287674188613892,
+      "step": 830
+    },
+    {
+      "epoch": 0.12365670543206242,
+      "grad_norm": 119.79529421225497,
+      "learning_rate": 4.991553122064028e-07,
+      "logits/chosen": -2.269420623779297,
+      "logits/rejected": -2.1465039253234863,
+      "logps/chosen": -460.19476318359375,
+      "logps/rejected": -437.865966796875,
+      "loss": 0.6107,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.6137462854385376,
+      "rewards/margins": 0.4687502980232239,
+      "rewards/rejected": -2.082496404647827,
+      "step": 840
+    },
+    {
+      "epoch": 0.1251288090681584,
+      "grad_norm": 75.9177588654981,
+      "learning_rate": 4.990464959355464e-07,
+      "logits/chosen": -2.3251051902770996,
+      "logits/rejected": -2.373628616333008,
+      "logps/chosen": -393.3048095703125,
+      "logps/rejected": -467.0416564941406,
+      "loss": 0.6201,
+      "rewards/accuracies": 0.5666666626930237,
+      "rewards/chosen": -1.5888453722000122,
+      "rewards/margins": 0.536378026008606,
+      "rewards/rejected": -2.125223159790039,
+      "step": 850
+    },
+    {
+      "epoch": 0.1266009127042544,
+      "grad_norm": 112.05360199697016,
+      "learning_rate": 4.98931102019645e-07,
+      "logits/chosen": -2.314889907836914,
+      "logits/rejected": -2.2704579830169678,
+      "logps/chosen": -329.2564392089844,
+      "logps/rejected": -392.57867431640625,
+      "loss": 0.5425,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.022720217704773,
+      "rewards/margins": 0.9595369100570679,
+      "rewards/rejected": -1.9822572469711304,
+      "step": 860
+    },
+    {
+      "epoch": 0.12807301634035037,
+      "grad_norm": 90.50989484982753,
+      "learning_rate": 4.988091335064037e-07,
+      "logits/chosen": -2.290186643600464,
+      "logits/rejected": -2.206688642501831,
+      "logps/chosen": -394.3182067871094,
+      "logps/rejected": -440.935546875,
+      "loss": 0.6308,
+      "rewards/accuracies": 0.7333332300186157,
+      "rewards/chosen": -1.573823094367981,
+      "rewards/margins": 0.5901575088500977,
+      "rewards/rejected": -2.163980722427368,
+      "step": 870
+    },
+    {
+      "epoch": 0.12954511997644635,
+      "grad_norm": 58.388436790562935,
+      "learning_rate": 4.98680593617171e-07,
+      "logits/chosen": -2.488431453704834,
+      "logits/rejected": -2.4395880699157715,
+      "logps/chosen": -386.14056396484375,
+      "logps/rejected": -313.35498046875,
+      "loss": 0.591,
+      "rewards/accuracies": 0.5666666626930237,
+      "rewards/chosen": -0.9596444964408875,
+      "rewards/margins": 0.1092836856842041,
+      "rewards/rejected": -1.0689282417297363,
+      "step": 880
+    },
+    {
+      "epoch": 0.1310172236125423,
+      "grad_norm": 40.99851316434281,
+      "learning_rate": 4.985454857468542e-07,
+      "logits/chosen": -2.243323802947998,
+      "logits/rejected": -2.22050404548645,
+      "logps/chosen": -432.0625,
+      "logps/rejected": -431.6446228027344,
+      "loss": 0.6383,
+      "rewards/accuracies": 0.5666667222976685,
+      "rewards/chosen": -1.2183120250701904,
+      "rewards/margins": 0.3802083432674408,
+      "rewards/rejected": -1.5985205173492432,
+      "step": 890
+    },
+    {
+      "epoch": 0.1324893272486383,
+      "grad_norm": 99.22369803083447,
+      "learning_rate": 4.984038134638297e-07,
+      "logits/chosen": -2.307290554046631,
+      "logits/rejected": -2.3129024505615234,
+      "logps/chosen": -355.7754821777344,
+      "logps/rejected": -406.7025146484375,
+      "loss": 0.5693,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0389937162399292,
+      "rewards/margins": 0.5364495515823364,
+      "rewards/rejected": -1.5754432678222656,
+      "step": 900
+    },
+    {
+      "epoch": 0.13396143088473428,
+      "grad_norm": 68.23159256155962,
+      "learning_rate": 4.982555805098483e-07,
+      "logits/chosen": -2.2068724632263184,
+      "logits/rejected": -2.078933000564575,
+      "logps/chosen": -337.3310546875,
+      "logps/rejected": -306.7698669433594,
+      "loss": 0.5051,
+      "rewards/accuracies": 0.5999999642372131,
+      "rewards/chosen": -0.9214721918106079,
+      "rewards/margins": 0.4675617814064026,
+      "rewards/rejected": -1.3890339136123657,
+      "step": 910
+    },
+    {
+      "epoch": 0.13543353452083026,
+      "grad_norm": 182.81778083629945,
+      "learning_rate": 4.981007907999372e-07,
+      "logits/chosen": -2.19437837600708,
+      "logits/rejected": -2.253805637359619,
+      "logps/chosen": -298.8421936035156,
+      "logps/rejected": -415.520751953125,
+      "loss": 0.6246,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": -1.0081114768981934,
+      "rewards/margins": 0.34868913888931274,
+      "rewards/rejected": -1.3568007946014404,
+      "step": 920
+    },
+    {
+      "epoch": 0.13690563815692625,
+      "grad_norm": 87.66168135410162,
+      "learning_rate": 4.979394484222961e-07,
+      "logits/chosen": -2.0963408946990967,
+      "logits/rejected": -2.127100944519043,
+      "logps/chosen": -277.10980224609375,
+      "logps/rejected": -451.10845947265625,
+      "loss": 0.5668,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -0.9092451333999634,
+      "rewards/margins": 0.9224964380264282,
+      "rewards/rejected": -1.8317416906356812,
+      "step": 930
+    },
+    {
+      "epoch": 0.13837774179302223,
+      "grad_norm": 113.42064742344525,
+      "learning_rate": 4.977715576381888e-07,
+      "logits/chosen": -2.062704563140869,
+      "logits/rejected": -2.0764923095703125,
+      "logps/chosen": -390.50482177734375,
+      "logps/rejected": -448.83880615234375,
+      "loss": 0.5976,
+      "rewards/accuracies": 0.7333332300186157,
+      "rewards/chosen": -1.1084699630737305,
+      "rewards/margins": 0.6334068775177002,
+      "rewards/rejected": -1.7418768405914307,
+      "step": 940
+    },
+    {
+      "epoch": 0.1398498454291182,
+      "grad_norm": 84.98755404610068,
+      "learning_rate": 4.975971228818315e-07,
+      "logits/chosen": -2.2141575813293457,
+      "logits/rejected": -2.1957879066467285,
+      "logps/chosen": -329.3928527832031,
+      "logps/rejected": -461.7449645996094,
+      "loss": 0.5695,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -1.0723673105239868,
+      "rewards/margins": 1.2911356687545776,
+      "rewards/rejected": -2.3635029792785645,
+      "step": 950
+    },
+    {
+      "epoch": 0.1413219490652142,
+      "grad_norm": 95.3981684051296,
+      "learning_rate": 4.974161487602753e-07,
+      "logits/chosen": -2.1243934631347656,
+      "logits/rejected": -2.1209025382995605,
+      "logps/chosen": -384.853759765625,
+      "logps/rejected": -452.0210876464844,
+      "loss": 0.5195,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.4263676404953003,
+      "rewards/margins": 0.6128391027450562,
+      "rewards/rejected": -2.0392069816589355,
+      "step": 960
+    },
+    {
+      "epoch": 0.14279405270131018,
+      "grad_norm": 113.96843076110557,
+      "learning_rate": 4.972286400532842e-07,
+      "logits/chosen": -2.0679757595062256,
+      "logits/rejected": -2.025749683380127,
+      "logps/chosen": -413.76727294921875,
+      "logps/rejected": -534.8135375976562,
+      "loss": 0.5322,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.8679730892181396,
+      "rewards/margins": 0.9302923083305359,
+      "rewards/rejected": -2.7982656955718994,
+      "step": 970
+    },
+    {
+      "epoch": 0.14426615633740617,
+      "grad_norm": 73.85476043164623,
+      "learning_rate": 4.970346017132097e-07,
+      "logits/chosen": -2.091668128967285,
+      "logits/rejected": -1.8952341079711914,
+      "logps/chosen": -463.33251953125,
+      "logps/rejected": -512.8320922851562,
+      "loss": 0.5526,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.806361436843872,
+      "rewards/margins": 1.1053744554519653,
+      "rewards/rejected": -2.9117355346679688,
+      "step": 980
+    },
+    {
+      "epoch": 0.14573825997350212,
+      "grad_norm": 76.9679999494502,
+      "learning_rate": 4.96834038864859e-07,
+      "logits/chosen": -2.069716215133667,
+      "logits/rejected": -2.039652109146118,
+      "logps/chosen": -477.3058166503906,
+      "logps/rejected": -492.8099060058594,
+      "loss": 0.4873,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -1.745233178138733,
+      "rewards/margins": 0.6889790296554565,
+      "rewards/rejected": -2.4342122077941895,
+      "step": 990
+    },
+    {
+      "epoch": 0.1472103636095981,
+      "grad_norm": 72.1161888821459,
+      "learning_rate": 4.966269568053605e-07,
+      "logits/chosen": -2.1215076446533203,
+      "logits/rejected": -2.054935932159424,
+      "logps/chosen": -439.8961486816406,
+      "logps/rejected": -526.8071899414062,
+      "loss": 0.5634,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.3309630155563354,
+      "rewards/margins": 0.9732717275619507,
+      "rewards/rejected": -2.3042349815368652,
+      "step": 1000
+    },
+    {
+      "epoch": 0.1486824672456941,
+      "grad_norm": 136.2850007298125,
+      "learning_rate": 4.964133610040232e-07,
+      "logits/chosen": -1.9484182596206665,
+      "logits/rejected": -1.8160221576690674,
+      "logps/chosen": -418.21173095703125,
+      "logps/rejected": -513.11767578125,
+      "loss": 0.5841,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.6914863586425781,
+      "rewards/margins": 0.8490256071090698,
+      "rewards/rejected": -2.5405116081237793,
+      "step": 1010
+    },
+    {
+      "epoch": 0.15015457088179007,
+      "grad_norm": 57.74832561411465,
+      "learning_rate": 4.961932571021928e-07,
+      "logits/chosen": -2.165980100631714,
+      "logits/rejected": -2.040670871734619,
+      "logps/chosen": -415.43768310546875,
+      "logps/rejected": -455.57171630859375,
+      "loss": 0.4488,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.5381168127059937,
+      "rewards/margins": 0.6904627084732056,
+      "rewards/rejected": -2.228579521179199,
+      "step": 1020
+    },
+    {
+      "epoch": 0.15162667451788606,
+      "grad_norm": 81.18313465845735,
+      "learning_rate": 4.959666509131025e-07,
+      "logits/chosen": -2.122326374053955,
+      "logits/rejected": -1.9732322692871094,
+      "logps/chosen": -398.5821228027344,
+      "logps/rejected": -463.12890625,
+      "loss": 0.5122,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.3551348447799683,
+      "rewards/margins": 1.0280563831329346,
+      "rewards/rejected": -2.3831915855407715,
+      "step": 1030
+    },
+    {
+      "epoch": 0.15309877815398204,
+      "grad_norm": 115.82876072399974,
+      "learning_rate": 4.957335484217193e-07,
+      "logits/chosen": -2.1244285106658936,
+      "logits/rejected": -2.145282745361328,
+      "logps/chosen": -412.422607421875,
+      "logps/rejected": -458.42578125,
+      "loss": 0.6047,
+      "rewards/accuracies": 0.5333333015441895,
+      "rewards/chosen": -1.257176160812378,
+      "rewards/margins": 0.412966787815094,
+      "rewards/rejected": -1.6701428890228271,
+      "step": 1040
+    },
+    {
+      "epoch": 0.15457088179007802,
+      "grad_norm": 85.64937174321328,
+      "learning_rate": 4.954939557845862e-07,
+      "logits/chosen": -2.214097499847412,
+      "logits/rejected": -2.086029529571533,
+      "logps/chosen": -468.0782165527344,
+      "logps/rejected": -518.0413208007812,
+      "loss": 0.6009,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4204020500183105,
+      "rewards/margins": 0.45726943016052246,
+      "rewards/rejected": -1.8776715993881226,
+      "step": 1050
+    },
+    {
+      "epoch": 0.156042985426174,
+      "grad_norm": 76.16188074968915,
+      "learning_rate": 4.952478793296594e-07,
+      "logits/chosen": -1.9984462261199951,
+      "logits/rejected": -2.0089707374572754,
+      "logps/chosen": -291.55047607421875,
+      "logps/rejected": -413.54534912109375,
+      "loss": 0.4861,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -1.1199281215667725,
+      "rewards/margins": 0.9721947908401489,
+      "rewards/rejected": -2.092123031616211,
+      "step": 1060
+    },
+    {
+      "epoch": 0.15751508906227,
+      "grad_norm": 66.89251406016741,
+      "learning_rate": 4.949953255561411e-07,
+      "logits/chosen": -2.1272060871124268,
+      "logits/rejected": -2.082313060760498,
+      "logps/chosen": -413.8898010253906,
+      "logps/rejected": -489.11163330078125,
+      "loss": 0.4714,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.1104748249053955,
+      "rewards/margins": 1.042687177658081,
+      "rewards/rejected": -2.1531615257263184,
+      "step": 1070
+    },
+    {
+      "epoch": 0.15898719269836598,
+      "grad_norm": 62.60189813831605,
+      "learning_rate": 4.947363011343083e-07,
+      "logits/chosen": -2.0016605854034424,
+      "logits/rejected": -1.9395275115966797,
+      "logps/chosen": -431.47515869140625,
+      "logps/rejected": -456.5384216308594,
+      "loss": 0.4674,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.4982682466506958,
+      "rewards/margins": 0.6886683106422424,
+      "rewards/rejected": -2.186936855316162,
+      "step": 1080
+    },
+    {
+      "epoch": 0.16045929633446196,
+      "grad_norm": 85.8807908568554,
+      "learning_rate": 4.944708129053362e-07,
+      "logits/chosen": -1.873248815536499,
+      "logits/rejected": -1.8893684148788452,
+      "logps/chosen": -446.6443786621094,
+      "logps/rejected": -435.57025146484375,
+      "loss": 0.5677,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.6512603759765625,
+      "rewards/margins": 0.6568886041641235,
+      "rewards/rejected": -2.3081488609313965,
+      "step": 1090
+    },
+    {
+      "epoch": 0.16193139997055792,
+      "grad_norm": 91.37167049438412,
+      "learning_rate": 4.941988678811176e-07,
+      "logits/chosen": -1.877694845199585,
+      "logits/rejected": -1.7398853302001953,
+      "logps/chosen": -440.3699645996094,
+      "logps/rejected": -484.8650817871094,
+      "loss": 0.5656,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.721146583557129,
+      "rewards/margins": 0.9025732278823853,
+      "rewards/rejected": -2.6237196922302246,
+      "step": 1100
+    },
+    {
+      "epoch": 0.1634035036066539,
+      "grad_norm": 70.1640832487081,
+      "learning_rate": 4.939204732440777e-07,
+      "logits/chosen": -1.9863033294677734,
+      "logits/rejected": -1.9010241031646729,
+      "logps/chosen": -374.4677734375,
+      "logps/rejected": -389.5589294433594,
+      "loss": 0.469,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.4531137943267822,
+      "rewards/margins": 0.8062105178833008,
+      "rewards/rejected": -2.259324312210083,
+      "step": 1110
+    },
+    {
+      "epoch": 0.16487560724274988,
+      "grad_norm": 83.51661410069474,
+      "learning_rate": 4.936356363469845e-07,
+      "logits/chosen": -1.878106713294983,
+      "logits/rejected": -1.8452297449111938,
+      "logps/chosen": -426.2998962402344,
+      "logps/rejected": -505.9771423339844,
+      "loss": 0.6071,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5932050943374634,
+      "rewards/margins": 0.5873239040374756,
+      "rewards/rejected": -2.1805288791656494,
+      "step": 1120
+    },
+    {
+      "epoch": 0.16634771087884587,
+      "grad_norm": 194.64662554460307,
+      "learning_rate": 4.933443647127546e-07,
+      "logits/chosen": -1.9042288064956665,
+      "logits/rejected": -1.8562898635864258,
+      "logps/chosen": -391.3512268066406,
+      "logps/rejected": -425.87518310546875,
+      "loss": 0.585,
+      "rewards/accuracies": 0.7333332300186157,
+      "rewards/chosen": -1.4531123638153076,
+      "rewards/margins": 0.6793515086174011,
+      "rewards/rejected": -2.1324636936187744,
+      "step": 1130
+    },
+    {
+      "epoch": 0.16781981451494185,
+      "grad_norm": 150.4762925491126,
+      "learning_rate": 4.930466660342543e-07,
+      "logits/chosen": -1.6633784770965576,
+      "logits/rejected": -1.5810251235961914,
+      "logps/chosen": -470.38555908203125,
+      "logps/rejected": -486.028076171875,
+      "loss": 0.6363,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -2.476055860519409,
+      "rewards/margins": 0.576412558555603,
+      "rewards/rejected": -3.0524685382843018,
+      "step": 1140
+    },
+    {
+      "epoch": 0.16929191815103783,
+      "grad_norm": 1059.8004323037026,
+      "learning_rate": 4.927425481740968e-07,
+      "logits/chosen": -1.704079031944275,
+      "logits/rejected": -1.6063989400863647,
+      "logps/chosen": -494.673095703125,
+      "logps/rejected": -524.99755859375,
+      "loss": 0.6717,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -2.659291982650757,
+      "rewards/margins": 0.5076649785041809,
+      "rewards/rejected": -3.166957139968872,
+      "step": 1150
+    },
+    {
+      "epoch": 0.17076402178713382,
+      "grad_norm": 92.60206409904589,
+      "learning_rate": 4.924320191644341e-07,
+      "logits/chosen": -1.8932876586914062,
+      "logits/rejected": -1.8256971836090088,
+      "logps/chosen": -526.1300048828125,
+      "logps/rejected": -605.43017578125,
+      "loss": 0.5503,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.274585008621216,
+      "rewards/margins": 1.0169695615768433,
+      "rewards/rejected": -3.2915549278259277,
+      "step": 1160
+    },
+    {
+      "epoch": 0.1722361254232298,
+      "grad_norm": 117.8554758287773,
+      "learning_rate": 4.921150872067452e-07,
+      "logits/chosen": -1.9694545269012451,
+      "logits/rejected": -1.8143869638442993,
+      "logps/chosen": -479.6485290527344,
+      "logps/rejected": -476.7415466308594,
+      "loss": 0.5049,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": -1.804478406906128,
+      "rewards/margins": 0.553841233253479,
+      "rewards/rejected": -2.3583197593688965,
+      "step": 1170
+    },
+    {
+      "epoch": 0.17370822905932579,
+      "grad_norm": 287.3277348516987,
+      "learning_rate": 4.917917606716194e-07,
+      "logits/chosen": -1.7659416198730469,
+      "logits/rejected": -1.7585029602050781,
+      "logps/chosen": -375.60015869140625,
+      "logps/rejected": -470.989013671875,
+      "loss": 0.5279,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.9602396488189697,
+      "rewards/margins": 0.42130643129348755,
+      "rewards/rejected": -2.3815460205078125,
+      "step": 1180
+    },
+    {
+      "epoch": 0.17518033269542177,
+      "grad_norm": 52.304859212281535,
+      "learning_rate": 4.914620480985352e-07,
+      "logits/chosen": -1.9349756240844727,
+      "logits/rejected": -1.9279472827911377,
+      "logps/chosen": -495.8687438964844,
+      "logps/rejected": -537.65283203125,
+      "loss": 0.5704,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.5462409257888794,
+      "rewards/margins": 0.8103898763656616,
+      "rewards/rejected": -2.356630563735962,
+      "step": 1190
+    },
+    {
+      "epoch": 0.17665243633151773,
+      "grad_norm": 91.8676158422065,
+      "learning_rate": 4.911259581956345e-07,
+      "logits/chosen": -1.833195686340332,
+      "logits/rejected": -1.9040639400482178,
+      "logps/chosen": -432.99395751953125,
+      "logps/rejected": -610.0288696289062,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.5811357498168945,
+      "rewards/margins": 0.8339434862136841,
+      "rewards/rejected": -2.415079116821289,
+      "step": 1200
+    },
+    {
+      "epoch": 0.1781245399676137,
+      "grad_norm": 79.47387367101264,
+      "learning_rate": 4.907834998394932e-07,
+      "logits/chosen": -1.9623525142669678,
+      "logits/rejected": -1.9679954051971436,
+      "logps/chosen": -290.91845703125,
+      "logps/rejected": -416.90032958984375,
+      "loss": 0.4977,
+      "rewards/accuracies": 0.9333332777023315,
+      "rewards/chosen": -0.8924083709716797,
+      "rewards/margins": 0.9155305027961731,
+      "rewards/rejected": -1.8079389333724976,
+      "step": 1210
+    },
+    {
+      "epoch": 0.1795966436037097,
+      "grad_norm": 96.05780834655081,
+      "learning_rate": 4.904346820748862e-07,
+      "logits/chosen": -1.9771655797958374,
+      "logits/rejected": -1.8380553722381592,
+      "logps/chosen": -442.7921447753906,
+      "logps/rejected": -441.2320861816406,
+      "loss": 0.5562,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5602648258209229,
+      "rewards/margins": 0.7709900140762329,
+      "rewards/rejected": -2.331254720687866,
+      "step": 1220
+    },
+    {
+      "epoch": 0.18106874723980568,
+      "grad_norm": 192.01556582915327,
+      "learning_rate": 4.900795141145487e-07,
+      "logits/chosen": -1.9421284198760986,
+      "logits/rejected": -1.789272665977478,
+      "logps/chosen": -334.18157958984375,
+      "logps/rejected": -461.389404296875,
+      "loss": 0.4702,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -1.4846737384796143,
+      "rewards/margins": 0.8886607885360718,
+      "rewards/rejected": -2.3733344078063965,
+      "step": 1230
+    },
+    {
+      "epoch": 0.18254085087590166,
+      "grad_norm": 255.85495023538195,
+      "learning_rate": 4.897180053389332e-07,
+      "logits/chosen": -1.9627825021743774,
+      "logits/rejected": -1.8504955768585205,
+      "logps/chosen": -432.42242431640625,
+      "logps/rejected": -501.09393310546875,
+      "loss": 0.5592,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9151782989501953,
+      "rewards/margins": 0.9223679304122925,
+      "rewards/rejected": -2.8375463485717773,
+      "step": 1240
+    },
+    {
+      "epoch": 0.18401295451199765,
+      "grad_norm": 100.32293673239501,
+      "learning_rate": 4.89350165295961e-07,
+      "logits/chosen": -1.9722315073013306,
+      "logits/rejected": -2.004082441329956,
+      "logps/chosen": -436.787109375,
+      "logps/rejected": -542.9174194335938,
+      "loss": 0.5656,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.8804203271865845,
+      "rewards/margins": 0.7965079545974731,
+      "rewards/rejected": -2.6769282817840576,
+      "step": 1250
+    },
+    {
+      "epoch": 0.18548505814809363,
+      "grad_norm": 108.56990565880037,
+      "learning_rate": 4.88976003700771e-07,
+      "logits/chosen": -2.0546443462371826,
+      "logits/rejected": -1.9407790899276733,
+      "logps/chosen": -571.1334228515625,
+      "logps/rejected": -592.9005126953125,
+      "loss": 0.5734,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.21150541305542,
+      "rewards/margins": 0.8162603378295898,
+      "rewards/rejected": -3.027766227722168,
+      "step": 1260
+    },
+    {
+      "epoch": 0.1869571617841896,
+      "grad_norm": 576.7575925067408,
+      "learning_rate": 4.885955304354622e-07,
+      "logits/chosen": -1.8984283208847046,
+      "logits/rejected": -1.8702316284179688,
+      "logps/chosen": -377.49481201171875,
+      "logps/rejected": -463.9615173339844,
+      "loss": 0.5305,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.3347193002700806,
+      "rewards/margins": 0.9362874031066895,
+      "rewards/rejected": -2.2710065841674805,
+      "step": 1270
+    },
+    {
+      "epoch": 0.1884292654202856,
+      "grad_norm": 73.36266299470105,
+      "learning_rate": 4.882087555488331e-07,
+      "logits/chosen": -2.1328797340393066,
+      "logits/rejected": -2.031864881515503,
+      "logps/chosen": -429.73828125,
+      "logps/rejected": -418.3163146972656,
+      "loss": 0.5371,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.1759955883026123,
+      "rewards/margins": 0.7771323919296265,
+      "rewards/rejected": -1.9531276226043701,
+      "step": 1280
+    },
+    {
+      "epoch": 0.18990136905638158,
+      "grad_norm": 68.02291613118399,
+      "learning_rate": 4.878156892561167e-07,
+      "logits/chosen": -2.009929895401001,
+      "logits/rejected": -1.9555679559707642,
+      "logps/chosen": -360.5475158691406,
+      "logps/rejected": -464.4381408691406,
+      "loss": 0.5558,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.6077797412872314,
+      "rewards/margins": 0.6679785251617432,
+      "rewards/rejected": -2.2757582664489746,
+      "step": 1290
+    },
+    {
+      "epoch": 0.19137347269247756,
+      "grad_norm": 168.9729209419952,
+      "learning_rate": 4.874163419387099e-07,
+      "logits/chosen": -1.9611870050430298,
+      "logits/rejected": -1.9176820516586304,
+      "logps/chosen": -349.78619384765625,
+      "logps/rejected": -480.7174377441406,
+      "loss": 0.5364,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.621067762374878,
+      "rewards/margins": 0.984261155128479,
+      "rewards/rejected": -2.6053287982940674,
+      "step": 1300
+    },
+    {
+      "epoch": 0.19284557632857352,
+      "grad_norm": 112.89657841963704,
+      "learning_rate": 4.870107241438999e-07,
+      "logits/chosen": -1.9834076166152954,
+      "logits/rejected": -1.9171555042266846,
+      "logps/chosen": -473.7816467285156,
+      "logps/rejected": -575.8986206054688,
+      "loss": 0.6297,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -2.2061500549316406,
+      "rewards/margins": 1.0230388641357422,
+      "rewards/rejected": -3.229188919067383,
+      "step": 1310
+    },
+    {
+      "epoch": 0.1943176799646695,
+      "grad_norm": 106.24775303317578,
+      "learning_rate": 4.865988465845852e-07,
+      "logits/chosen": -1.9366060495376587,
+      "logits/rejected": -1.8908555507659912,
+      "logps/chosen": -482.27618408203125,
+      "logps/rejected": -524.3118286132812,
+      "loss": 0.5819,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -2.2628026008605957,
+      "rewards/margins": 0.5921173095703125,
+      "rewards/rejected": -2.854919910430908,
+      "step": 1320
+    },
+    {
+      "epoch": 0.1957897836007655,
+      "grad_norm": 52.229401388486096,
+      "learning_rate": 4.861807201389933e-07,
+      "logits/chosen": -2.0587615966796875,
+      "logits/rejected": -1.9710086584091187,
+      "logps/chosen": -440.68341064453125,
+      "logps/rejected": -486.0523986816406,
+      "loss": 0.5216,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7683641910552979,
+      "rewards/margins": 0.9764081835746765,
+      "rewards/rejected": -2.7447726726531982,
+      "step": 1330
+    },
+    {
+      "epoch": 0.19726188723686147,
+      "grad_norm": 102.49039584777856,
+      "learning_rate": 4.857563558503925e-07,
+      "logits/chosen": -1.9479601383209229,
+      "logits/rejected": -1.9149713516235352,
+      "logps/chosen": -427.7066955566406,
+      "logps/rejected": -524.1619873046875,
+      "loss": 0.5739,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.6376683712005615,
+      "rewards/margins": 0.7223914861679077,
+      "rewards/rejected": -2.3600597381591797,
+      "step": 1340
+    },
+    {
+      "epoch": 0.19873399087295746,
+      "grad_norm": 86.03021132265596,
+      "learning_rate": 4.853257649268014e-07,
+      "logits/chosen": -1.8271957635879517,
+      "logits/rejected": -1.7711585760116577,
+      "logps/chosen": -387.6052551269531,
+      "logps/rejected": -394.2628173828125,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.6212457418441772,
+      "rewards/margins": 0.5990225672721863,
+      "rewards/rejected": -2.2202682495117188,
+      "step": 1350
+    },
+    {
+      "epoch": 0.20020609450905344,
+      "grad_norm": 95.9549443531512,
+      "learning_rate": 4.848889587406915e-07,
+      "logits/chosen": -1.9340747594833374,
+      "logits/rejected": -2.000556468963623,
+      "logps/chosen": -424.41619873046875,
+      "logps/rejected": -503.3523864746094,
+      "loss": 0.5768,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.6836246252059937,
+      "rewards/margins": 0.7771691083908081,
+      "rewards/rejected": -2.460793972015381,
+      "step": 1360
+    },
+    {
+      "epoch": 0.20167819814514942,
+      "grad_norm": 66.58619020064566,
+      "learning_rate": 4.84445948828688e-07,
+      "logits/chosen": -1.7994775772094727,
+      "logits/rejected": -1.6953080892562866,
+      "logps/chosen": -459.05084228515625,
+      "logps/rejected": -351.1479187011719,
+      "loss": 0.7021,
+      "rewards/accuracies": 0.46666663885116577,
+      "rewards/chosen": -1.83424973487854,
+      "rewards/margins": 0.1528480052947998,
+      "rewards/rejected": -1.9870975017547607,
+      "step": 1370
+    },
+    {
+      "epoch": 0.2031503017812454,
+      "grad_norm": 39.301434660219456,
+      "learning_rate": 4.839967468912645e-07,
+      "logits/chosen": -2.0798094272613525,
+      "logits/rejected": -2.029099464416504,
+      "logps/chosen": -426.1573181152344,
+      "logps/rejected": -466.2427673339844,
+      "loss": 0.5844,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.2144460678100586,
+      "rewards/margins": 0.6685677766799927,
+      "rewards/rejected": -1.8830139636993408,
+      "step": 1380
+    },
+    {
+      "epoch": 0.2046224054173414,
+      "grad_norm": 78.71981934177275,
+      "learning_rate": 4.83541364792434e-07,
+      "logits/chosen": -1.8970882892608643,
+      "logits/rejected": -1.9339784383773804,
+      "logps/chosen": -379.80426025390625,
+      "logps/rejected": -446.1246032714844,
+      "loss": 0.5298,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.593406319618225,
+      "rewards/margins": 0.5336878895759583,
+      "rewards/rejected": -2.127094268798828,
+      "step": 1390
+    },
+    {
+      "epoch": 0.20609450905343737,
+      "grad_norm": 107.64488455114477,
+      "learning_rate": 4.83079814559436e-07,
+      "logits/chosen": -1.9898523092269897,
+      "logits/rejected": -1.8959048986434937,
+      "logps/chosen": -463.18060302734375,
+      "logps/rejected": -483.04534912109375,
+      "loss": 0.5633,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.7604682445526123,
+      "rewards/margins": 0.6334084868431091,
+      "rewards/rejected": -2.393876552581787,
+      "step": 1400
+    },
+    {
+      "epoch": 0.20756661268953333,
+      "grad_norm": 57.38521937426867,
+      "learning_rate": 4.826121083824181e-07,
+      "logits/chosen": -1.8779270648956299,
+      "logits/rejected": -1.8601785898208618,
+      "logps/chosen": -461.5616760253906,
+      "logps/rejected": -491.1333923339844,
+      "loss": 0.5071,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.7569482326507568,
+      "rewards/margins": 0.6989758610725403,
+      "rewards/rejected": -2.4559242725372314,
+      "step": 1410
+    },
+    {
+      "epoch": 0.20903871632562931,
+      "grad_norm": 70.40701427850496,
+      "learning_rate": 4.82138258614115e-07,
+      "logits/chosen": -1.9726396799087524,
+      "logits/rejected": -1.910292625427246,
+      "logps/chosen": -437.85479736328125,
+      "logps/rejected": -490.97698974609375,
+      "loss": 0.5426,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.5512851476669312,
+      "rewards/margins": 0.6094131469726562,
+      "rewards/rejected": -2.160698175430298,
+      "step": 1420
+    },
+    {
+      "epoch": 0.2105108199617253,
+      "grad_norm": 68.54955508063232,
+      "learning_rate": 4.816582777695212e-07,
+      "logits/chosen": -2.0624613761901855,
+      "logits/rejected": -1.977168321609497,
+      "logps/chosen": -336.0431213378906,
+      "logps/rejected": -433.075439453125,
+      "loss": 0.5154,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.1242858171463013,
+      "rewards/margins": 0.7905791401863098,
+      "rewards/rejected": -1.9148648977279663,
+      "step": 1430
+    },
+    {
+      "epoch": 0.21198292359782128,
+      "grad_norm": 53.662406903457814,
+      "learning_rate": 4.811721785255612e-07,
+      "logits/chosen": -1.8410117626190186,
+      "logits/rejected": -1.9229590892791748,
+      "logps/chosen": -330.0913391113281,
+      "logps/rejected": -383.91876220703125,
+      "loss": 0.5438,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.4411814212799072,
+      "rewards/margins": 0.48734864592552185,
+      "rewards/rejected": -1.9285300970077515,
+      "step": 1440
+    },
+    {
+      "epoch": 0.21345502723391727,
+      "grad_norm": 99.78983887665171,
+      "learning_rate": 4.806799737207546e-07,
+      "logits/chosen": -1.7521324157714844,
+      "logits/rejected": -1.763819932937622,
+      "logps/chosen": -397.3249206542969,
+      "logps/rejected": -485.8280334472656,
+      "loss": 0.5558,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8569271564483643,
+      "rewards/margins": 0.6991360187530518,
+      "rewards/rejected": -2.556062936782837,
+      "step": 1450
+    },
+    {
+      "epoch": 0.21492713087001325,
+      "grad_norm": 77.4751921400414,
+      "learning_rate": 4.801816763548766e-07,
+      "logits/chosen": -2.006967067718506,
+      "logits/rejected": -1.9145374298095703,
+      "logps/chosen": -424.72161865234375,
+      "logps/rejected": -503.0011291503906,
+      "loss": 0.5403,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.3916488885879517,
+      "rewards/margins": 0.8669332265853882,
+      "rewards/rejected": -2.258582353591919,
+      "step": 1460
+    },
+    {
+      "epoch": 0.21639923450610923,
+      "grad_norm": 106.09643165111075,
+      "learning_rate": 4.796772995886151e-07,
+      "logits/chosen": -1.9247760772705078,
+      "logits/rejected": -1.9768764972686768,
+      "logps/chosen": -395.3624572753906,
+      "logps/rejected": -427.5353088378906,
+      "loss": 0.5898,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.313778042793274,
+      "rewards/margins": 0.5099066495895386,
+      "rewards/rejected": -1.8236846923828125,
+      "step": 1470
+    },
+    {
+      "epoch": 0.21787133814220522,
+      "grad_norm": 164.85826302580128,
+      "learning_rate": 4.791668567432229e-07,
+      "logits/chosen": -1.9292771816253662,
+      "logits/rejected": -1.7980287075042725,
+      "logps/chosen": -425.4931640625,
+      "logps/rejected": -439.97747802734375,
+      "loss": 0.616,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.306755781173706,
+      "rewards/margins": 0.6289185285568237,
+      "rewards/rejected": -1.9356744289398193,
+      "step": 1480
+    },
+    {
+      "epoch": 0.2193434417783012,
+      "grad_norm": 145.10508377533765,
+      "learning_rate": 4.78650361300166e-07,
+      "logits/chosen": -1.8893663883209229,
+      "logits/rejected": -1.882112741470337,
+      "logps/chosen": -441.1393127441406,
+      "logps/rejected": -517.1280517578125,
+      "loss": 0.4683,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.0786323547363281,
+      "rewards/margins": 0.9214528799057007,
+      "rewards/rejected": -2.0000853538513184,
+      "step": 1490
+    },
+    {
+      "epoch": 0.22081554541439719,
+      "grad_norm": 198.00339167625546,
+      "learning_rate": 4.781278269007675e-07,
+      "logits/chosen": -1.9149973392486572,
+      "logits/rejected": -1.761517882347107,
+      "logps/chosen": -499.5852966308594,
+      "logps/rejected": -563.583984375,
+      "loss": 0.4889,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.650298833847046,
+      "rewards/margins": 1.0565168857574463,
+      "rewards/rejected": -2.7068159580230713,
+      "step": 1500
+    },
+    {
+      "epoch": 0.22228764905049314,
+      "grad_norm": 49.036813921932804,
+      "learning_rate": 4.775992673458469e-07,
+      "logits/chosen": -1.8019440174102783,
+      "logits/rejected": -1.7608954906463623,
+      "logps/chosen": -461.46319580078125,
+      "logps/rejected": -509.35504150390625,
+      "loss": 0.4643,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -1.8942543268203735,
+      "rewards/margins": 0.8434314727783203,
+      "rewards/rejected": -2.7376856803894043,
+      "step": 1510
+    },
+    {
+      "epoch": 0.22375975268658913,
+      "grad_norm": 47.1731224324246,
+      "learning_rate": 4.770646965953564e-07,
+      "logits/chosen": -1.7631785869598389,
+      "logits/rejected": -1.7487800121307373,
+      "logps/chosen": -428.530517578125,
+      "logps/rejected": -518.99365234375,
+      "loss": 0.6563,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.326993703842163,
+      "rewards/margins": 1.1345717906951904,
+      "rewards/rejected": -2.4615654945373535,
+      "step": 1520
+    },
+    {
+      "epoch": 0.2252318563226851,
+      "grad_norm": 90.82425083212647,
+      "learning_rate": 4.765241287680116e-07,
+      "logits/chosen": -2.0718226432800293,
+      "logits/rejected": -1.8782758712768555,
+      "logps/chosen": -428.19366455078125,
+      "logps/rejected": -390.31036376953125,
+      "loss": 0.512,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -0.919861912727356,
+      "rewards/margins": 0.9122627973556519,
+      "rewards/rejected": -1.8321247100830078,
+      "step": 1530
+    },
+    {
+      "epoch": 0.2267039599587811,
+      "grad_norm": 99.4352748279063,
+      "learning_rate": 4.759775781409187e-07,
+      "logits/chosen": -1.95816171169281,
+      "logits/rejected": -1.8278312683105469,
+      "logps/chosen": -447.49530029296875,
+      "logps/rejected": -474.17919921875,
+      "loss": 0.562,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -1.7144454717636108,
+      "rewards/margins": 0.7575380206108093,
+      "rewards/rejected": -2.4719834327697754,
+      "step": 1540
+    },
+    {
+      "epoch": 0.22817606359487708,
+      "grad_norm": 70.96061670177698,
+      "learning_rate": 4.7542505914919775e-07,
+      "logits/chosen": -1.9224542379379272,
+      "logits/rejected": -1.9225962162017822,
+      "logps/chosen": -454.31597900390625,
+      "logps/rejected": -521.6945190429688,
+      "loss": 0.4991,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.6472991704940796,
+      "rewards/margins": 1.0311214923858643,
+      "rewards/rejected": -2.6784205436706543,
+      "step": 1550
+    },
+    {
+      "epoch": 0.22964816723097306,
+      "grad_norm": 225.93570983559735,
+      "learning_rate": 4.7486658638560076e-07,
+      "logits/chosen": -1.9805772304534912,
+      "logits/rejected": -2.0407538414001465,
+      "logps/chosen": -398.05804443359375,
+      "logps/rejected": -469.94061279296875,
+      "loss": 0.5792,
+      "rewards/accuracies": 0.7333332300186157,
+      "rewards/chosen": -1.4981138706207275,
+      "rewards/margins": 0.6817944645881653,
+      "rewards/rejected": -2.179908275604248,
+      "step": 1560
+    },
+    {
+      "epoch": 0.23112027086706904,
+      "grad_norm": 75.77626864521106,
+      "learning_rate": 4.7430217460012703e-07,
+      "logits/chosen": -1.89935302734375,
+      "logits/rejected": -1.8430445194244385,
+      "logps/chosen": -416.69219970703125,
+      "logps/rejected": -476.912353515625,
+      "loss": 0.5618,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4287060499191284,
+      "rewards/margins": 1.1412321329116821,
+      "rewards/rejected": -2.5699384212493896,
+      "step": 1570
+    },
+    {
+      "epoch": 0.23259237450316503,
+      "grad_norm": 75.35179639832687,
+      "learning_rate": 4.7373183869963295e-07,
+      "logits/chosen": -2.1149773597717285,
+      "logits/rejected": -2.0849950313568115,
+      "logps/chosen": -438.9716796875,
+      "logps/rejected": -464.542724609375,
+      "loss": 0.5682,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": -1.0148811340332031,
+      "rewards/margins": 0.6320945024490356,
+      "rewards/rejected": -1.6469755172729492,
+      "step": 1580
+    },
+    {
+      "epoch": 0.234064478139261,
+      "grad_norm": 129.63107922689574,
+      "learning_rate": 4.7315559374743896e-07,
+      "logits/chosen": -2.100356101989746,
+      "logits/rejected": -2.1032233238220215,
+      "logps/chosen": -368.68450927734375,
+      "logps/rejected": -456.0458068847656,
+      "loss": 0.5552,
+      "rewards/accuracies": 0.6333332657814026,
+      "rewards/chosen": -1.142398476600647,
+      "rewards/margins": 0.6042104959487915,
+      "rewards/rejected": -1.746609091758728,
+      "step": 1590
+    },
+    {
+      "epoch": 0.235536581775357,
+      "grad_norm": 61.71219474707247,
+      "learning_rate": 4.725734549629308e-07,
+      "logits/chosen": -1.8276703357696533,
+      "logits/rejected": -1.7567144632339478,
+      "logps/chosen": -372.30694580078125,
+      "logps/rejected": -504.8480529785156,
+      "loss": 0.5147,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.3657010793685913,
+      "rewards/margins": 1.2297759056091309,
+      "rewards/rejected": -2.5954768657684326,
+      "step": 1600
+    },
+    {
+      "epoch": 0.23700868541145298,
+      "grad_norm": 81.0493165852162,
+      "learning_rate": 4.719854377211585e-07,
+      "logits/chosen": -1.9518165588378906,
+      "logits/rejected": -1.951531171798706,
+      "logps/chosen": -387.6022033691406,
+      "logps/rejected": -385.2184143066406,
+      "loss": 0.5937,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.3865382671356201,
+      "rewards/margins": 0.5220063328742981,
+      "rewards/rejected": -1.9085447788238525,
+      "step": 1610
+    },
+    {
+      "epoch": 0.23848078904754894,
+      "grad_norm": 60.09382845884993,
+      "learning_rate": 4.713915575524296e-07,
+      "logits/chosen": -2.0600926876068115,
+      "logits/rejected": -2.0641098022460938,
+      "logps/chosen": -313.11114501953125,
+      "logps/rejected": -348.462158203125,
+      "loss": 0.6013,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.1776988506317139,
+      "rewards/margins": 0.34402555227279663,
+      "rewards/rejected": -1.5217244625091553,
+      "step": 1620
+    },
+    {
+      "epoch": 0.23995289268364492,
+      "grad_norm": 55.96151058915186,
+      "learning_rate": 4.7079183014189937e-07,
+      "logits/chosen": -1.9451329708099365,
+      "logits/rejected": -1.8646180629730225,
+      "logps/chosen": -323.1206970214844,
+      "logps/rejected": -387.49273681640625,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -0.9625275731086731,
+      "rewards/margins": 0.8532989621162415,
+      "rewards/rejected": -1.815826654434204,
+      "step": 1630
+    },
+    {
+      "epoch": 0.2414249963197409,
+      "grad_norm": 75.74723272789755,
+      "learning_rate": 4.7018627132915634e-07,
+      "logits/chosen": -1.9486162662506104,
+      "logits/rejected": -1.874757170677185,
+      "logps/chosen": -378.52557373046875,
+      "logps/rejected": -382.94622802734375,
+      "loss": 0.528,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.0989223718643188,
+      "rewards/margins": 0.5968096256256104,
+      "rewards/rejected": -1.6957321166992188,
+      "step": 1640
+    },
+    {
+      "epoch": 0.2428970999558369,
+      "grad_norm": 89.07340706477274,
+      "learning_rate": 4.695748971078042e-07,
+      "logits/chosen": -1.9029546976089478,
+      "logits/rejected": -1.7417056560516357,
+      "logps/chosen": -391.79150390625,
+      "logps/rejected": -363.08538818359375,
+      "loss": 0.5185,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.0512101650238037,
+      "rewards/margins": 0.5672429800033569,
+      "rewards/rejected": -1.6184532642364502,
+      "step": 1650
+    },
+    {
+      "epoch": 0.24436920359193287,
+      "grad_norm": 50.4856901474438,
+      "learning_rate": 4.689577236250389e-07,
+      "logits/chosen": -1.6937227249145508,
+      "logits/rejected": -1.5301963090896606,
+      "logps/chosen": -408.7888488769531,
+      "logps/rejected": -470.7008361816406,
+      "loss": 0.4977,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.4841692447662354,
+      "rewards/margins": 0.727135956287384,
+      "rewards/rejected": -2.2113051414489746,
+      "step": 1660
+    },
+    {
+      "epoch": 0.24584130722802885,
+      "grad_norm": 82.14693999635138,
+      "learning_rate": 4.683347671812228e-07,
+      "logits/chosen": -1.7227070331573486,
+      "logits/rejected": -1.527448296546936,
+      "logps/chosen": -418.6912536621094,
+      "logps/rejected": -401.7694396972656,
+      "loss": 0.5397,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -1.6010897159576416,
+      "rewards/margins": 0.7377594709396362,
+      "rewards/rejected": -2.3388493061065674,
+      "step": 1670
+    },
+    {
+      "epoch": 0.24731341086412484,
+      "grad_norm": 183.94121321876028,
+      "learning_rate": 4.677060442294537e-07,
+      "logits/chosen": -1.6719534397125244,
+      "logits/rejected": -1.6455955505371094,
+      "logps/chosen": -524.6824951171875,
+      "logps/rejected": -547.9179077148438,
+      "loss": 0.5941,
+      "rewards/accuracies": 0.5666667222976685,
+      "rewards/chosen": -2.1113953590393066,
+      "rewards/margins": 0.4821031987667084,
+      "rewards/rejected": -2.593498468399048,
+      "step": 1680
+    },
+    {
+      "epoch": 0.24878551450022082,
+      "grad_norm": 75.5082371575327,
+      "learning_rate": 4.6707157137513056e-07,
+      "logits/chosen": -1.74550461769104,
+      "logits/rejected": -1.7400407791137695,
+      "logps/chosen": -470.6490173339844,
+      "logps/rejected": -461.89605712890625,
+      "loss": 0.6451,
+      "rewards/accuracies": 0.6333333849906921,
+      "rewards/chosen": -1.9410581588745117,
+      "rewards/margins": 0.4430798888206482,
+      "rewards/rejected": -2.3841381072998047,
+      "step": 1690
+    },
+    {
+      "epoch": 0.2502576181363168,
+      "grad_norm": 98.82772188339317,
+      "learning_rate": 4.664313653755147e-07,
+      "logits/chosen": -1.9526565074920654,
+      "logits/rejected": -1.883766770362854,
+      "logps/chosen": -564.9190063476562,
+      "logps/rejected": -515.51806640625,
+      "loss": 0.5262,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6935169696807861,
+      "rewards/margins": 0.49616527557373047,
+      "rewards/rejected": -2.1896822452545166,
+      "step": 1700
+    },
+    {
+      "epoch": 0.25172972177241276,
+      "grad_norm": 101.66239188256434,
+      "learning_rate": 4.6578544313928735e-07,
+      "logits/chosen": -1.4847203493118286,
+      "logits/rejected": -1.3912432193756104,
+      "logps/chosen": -449.73388671875,
+      "logps/rejected": -569.7845458984375,
+      "loss": 0.5128,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.9837766885757446,
+      "rewards/margins": 1.1695940494537354,
+      "rewards/rejected": -3.1533703804016113,
+      "step": 1710
+    },
+    {
+      "epoch": 0.2532018254085088,
+      "grad_norm": 71.15192406825254,
+      "learning_rate": 4.6513382172610324e-07,
+      "logits/chosen": -1.4488019943237305,
+      "logits/rejected": -1.2987825870513916,
+      "logps/chosen": -476.8401794433594,
+      "logps/rejected": -549.9502563476562,
+      "loss": 0.5701,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.4032297134399414,
+      "rewards/margins": 0.8704252243041992,
+      "rewards/rejected": -3.2736549377441406,
+      "step": 1720
+    },
+    {
+      "epoch": 0.25467392904460473,
+      "grad_norm": 115.6657540517164,
+      "learning_rate": 4.6447651834613955e-07,
+      "logits/chosen": -1.5709898471832275,
+      "logits/rejected": -1.5908291339874268,
+      "logps/chosen": -520.153564453125,
+      "logps/rejected": -584.2728271484375,
+      "loss": 0.6331,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.6676547527313232,
+      "rewards/margins": 0.5557428598403931,
+      "rewards/rejected": -3.223397731781006,
+      "step": 1730
+    },
+    {
+      "epoch": 0.25614603268070074,
+      "grad_norm": 75.3319050439407,
+      "learning_rate": 4.638135503596419e-07,
+      "logits/chosen": -1.6912071704864502,
+      "logits/rejected": -1.7145217657089233,
+      "logps/chosen": -448.5433654785156,
+      "logps/rejected": -559.8304443359375,
+      "loss": 0.5967,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9018909931182861,
+      "rewards/margins": 0.6624494194984436,
+      "rewards/rejected": -2.564340353012085,
+      "step": 1740
+    },
+    {
+      "epoch": 0.2576181363167967,
+      "grad_norm": 135.1266647777451,
+      "learning_rate": 4.6314493527646553e-07,
+      "logits/chosen": -1.6531956195831299,
+      "logits/rejected": -1.5040241479873657,
+      "logps/chosen": -614.0111083984375,
+      "logps/rejected": -600.8482666015625,
+      "loss": 0.5588,
+      "rewards/accuracies": 0.5333333015441895,
+      "rewards/chosen": -2.88460636138916,
+      "rewards/margins": 0.49871939420700073,
+      "rewards/rejected": -3.3833255767822266,
+      "step": 1750
+    },
+    {
+      "epoch": 0.2590902399528927,
+      "grad_norm": 229.84979404838685,
+      "learning_rate": 4.624706907556129e-07,
+      "logits/chosen": -1.535413384437561,
+      "logits/rejected": -1.443834662437439,
+      "logps/chosen": -490.82684326171875,
+      "logps/rejected": -513.51220703125,
+      "loss": 0.6302,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -2.540764570236206,
+      "rewards/margins": 0.6940882802009583,
+      "rewards/rejected": -3.2348525524139404,
+      "step": 1760
+    },
+    {
+      "epoch": 0.26056234358898867,
+      "grad_norm": 102.39544835337084,
+      "learning_rate": 4.617908346047673e-07,
+      "logits/chosen": -1.5806939601898193,
+      "logits/rejected": -1.4859967231750488,
+      "logps/chosen": -411.6058044433594,
+      "logps/rejected": -524.4972534179688,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.6666667461395264,
+      "rewards/chosen": -1.9826698303222656,
+      "rewards/margins": 1.014257788658142,
+      "rewards/rejected": -2.9969279766082764,
+      "step": 1770
+    },
+    {
+      "epoch": 0.2620344472250846,
+      "grad_norm": 51.64233638006087,
+      "learning_rate": 4.6110538477982265e-07,
+      "logits/chosen": -1.85050368309021,
+      "logits/rejected": -1.7250585556030273,
+      "logps/chosen": -349.91790771484375,
+      "logps/rejected": -487.35943603515625,
+      "loss": 0.5408,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.2039263248443604,
+      "rewards/margins": 1.247125506401062,
+      "rewards/rejected": -2.451051712036133,
+      "step": 1780
+    },
+    {
+      "epoch": 0.26350655086118063,
+      "grad_norm": 161.90036469442958,
+      "learning_rate": 4.6041435938440887e-07,
+      "logits/chosen": -1.9583768844604492,
+      "logits/rejected": -1.820737600326538,
+      "logps/chosen": -329.48724365234375,
+      "logps/rejected": -395.01666259765625,
+      "loss": 0.5561,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.8706947565078735,
+      "rewards/margins": 0.616737961769104,
+      "rewards/rejected": -1.4874327182769775,
+      "step": 1790
+    },
+    {
+      "epoch": 0.2649786544972766,
+      "grad_norm": 68.37449519907922,
+      "learning_rate": 4.5971777666941445e-07,
+      "logits/chosen": -1.9366155862808228,
+      "logits/rejected": -1.9282124042510986,
+      "logps/chosen": -389.47503662109375,
+      "logps/rejected": -463.842529296875,
+      "loss": 0.629,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.09441077709198,
+      "rewards/margins": 0.9559955596923828,
+      "rewards/rejected": -2.0504064559936523,
+      "step": 1800
+    },
+    {
+      "epoch": 0.2664507581333726,
+      "grad_norm": 58.94192935579279,
+      "learning_rate": 4.5901565503250373e-07,
+      "logits/chosen": -1.8732668161392212,
+      "logits/rejected": -1.8085658550262451,
+      "logps/chosen": -410.6376037597656,
+      "logps/rejected": -477.9228515625,
+      "loss": 0.4429,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -1.1377919912338257,
+      "rewards/margins": 0.9135912656784058,
+      "rewards/rejected": -2.0513832569122314,
+      "step": 1810
+    },
+    {
+      "epoch": 0.26792286176946856,
+      "grad_norm": 106.17374938574147,
+      "learning_rate": 4.583080130176312e-07,
+      "logits/chosen": -1.6969807147979736,
+      "logits/rejected": -1.5779601335525513,
+      "logps/chosen": -423.90814208984375,
+      "logps/rejected": -438.09228515625,
+      "loss": 0.5464,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.6757621765136719,
+      "rewards/margins": 0.6260987520217896,
+      "rewards/rejected": -2.301860809326172,
+      "step": 1820
+    },
+    {
+      "epoch": 0.26939496540556457,
+      "grad_norm": 58.151446689539824,
+      "learning_rate": 4.575948693145518e-07,
+      "logits/chosen": -1.5030003786087036,
+      "logits/rejected": -1.5252426862716675,
+      "logps/chosen": -478.30322265625,
+      "logps/rejected": -531.8361206054688,
+      "loss": 0.4875,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": -1.7562452554702759,
+      "rewards/margins": 0.7899385690689087,
+      "rewards/rejected": -2.5461840629577637,
+      "step": 1830
+    },
+    {
+      "epoch": 0.2708670690416605,
+      "grad_norm": 140.9049693185743,
+      "learning_rate": 4.568762427583275e-07,
+      "logits/chosen": -1.4599277973175049,
+      "logits/rejected": -1.5029892921447754,
+      "logps/chosen": -467.74530029296875,
+      "logps/rejected": -566.2022705078125,
+      "loss": 0.5047,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -2.079827070236206,
+      "rewards/margins": 0.8469683527946472,
+      "rewards/rejected": -2.926795244216919,
+      "step": 1840
+    },
+    {
+      "epoch": 0.27233917267775654,
+      "grad_norm": 98.50078757323077,
+      "learning_rate": 4.561521523288293e-07,
+      "logits/chosen": -1.3252590894699097,
+      "logits/rejected": -1.3201786279678345,
+      "logps/chosen": -518.99072265625,
+      "logps/rejected": -627.2073364257812,
+      "loss": 0.5429,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.148075580596924,
+      "rewards/margins": 1.0770463943481445,
+      "rewards/rejected": -3.2251219749450684,
+      "step": 1850
+    },
+    {
+      "epoch": 0.2738112763138525,
+      "grad_norm": 77.53821488417238,
+      "learning_rate": 4.554226171502365e-07,
+      "logits/chosen": -1.7430492639541626,
+      "logits/rejected": -1.5309737920761108,
+      "logps/chosen": -462.879638671875,
+      "logps/rejected": -513.399169921875,
+      "loss": 0.4981,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.619969367980957,
+      "rewards/margins": 0.6844587922096252,
+      "rewards/rejected": -2.3044278621673584,
+      "step": 1860
+    },
+    {
+      "epoch": 0.2752833799499485,
+      "grad_norm": 47.10442635576484,
+      "learning_rate": 4.546876564905313e-07,
+      "logits/chosen": -1.619408369064331,
+      "logits/rejected": -1.840978980064392,
+      "logps/chosen": -378.6471862792969,
+      "logps/rejected": -593.4290771484375,
+      "loss": 0.5442,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.5935884714126587,
+      "rewards/margins": 0.9168362617492676,
+      "rewards/rejected": -2.5104246139526367,
+      "step": 1870
+    },
+    {
+      "epoch": 0.27675548358604446,
+      "grad_norm": 273.2367553228704,
+      "learning_rate": 4.5394728976099015e-07,
+      "logits/chosen": -1.5420583486557007,
+      "logits/rejected": -1.3446629047393799,
+      "logps/chosen": -455.2611389160156,
+      "logps/rejected": -630.8297119140625,
+      "loss": 0.4648,
+      "rewards/accuracies": 0.9000000953674316,
+      "rewards/chosen": -1.963801383972168,
+      "rewards/margins": 2.1008026599884033,
+      "rewards/rejected": -4.06460428237915,
+      "step": 1880
+    },
+    {
+      "epoch": 0.2782275872221404,
+      "grad_norm": 192.7634248185056,
+      "learning_rate": 4.532015365156705e-07,
+      "logits/chosen": -1.4305152893066406,
+      "logits/rejected": -1.4293017387390137,
+      "logps/chosen": -510.6460876464844,
+      "logps/rejected": -551.4142456054688,
+      "loss": 0.6284,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -2.2284276485443115,
+      "rewards/margins": 0.8585487604141235,
+      "rewards/rejected": -3.0869762897491455,
+      "step": 1890
+    },
+    {
+      "epoch": 0.2796996908582364,
+      "grad_norm": 57.69637029955735,
+      "learning_rate": 4.524504164508951e-07,
+      "logits/chosen": -1.6792455911636353,
+      "logits/rejected": -1.6008403301239014,
+      "logps/chosen": -463.2054138183594,
+      "logps/rejected": -509.00537109375,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.6925989389419556,
+      "rewards/margins": 0.686219334602356,
+      "rewards/rejected": -2.3788185119628906,
+      "step": 1900
+    },
+    {
+      "epoch": 0.2811717944943324,
+      "grad_norm": 68.40972228659973,
+      "learning_rate": 4.5169394940473137e-07,
+      "logits/chosen": -1.7620881795883179,
+      "logits/rejected": -1.6334224939346313,
+      "logps/chosen": -453.60333251953125,
+      "logps/rejected": -571.8778686523438,
+      "loss": 0.511,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.689611792564392,
+      "rewards/margins": 1.1878001689910889,
+      "rewards/rejected": -2.8774123191833496,
+      "step": 1910
+    },
+    {
+      "epoch": 0.2826438981304284,
+      "grad_norm": 108.28897894151297,
+      "learning_rate": 4.509321553564676e-07,
+      "logits/chosen": -1.6775577068328857,
+      "logits/rejected": -1.6379661560058594,
+      "logps/chosen": -444.2704162597656,
+      "logps/rejected": -509.41546630859375,
+      "loss": 0.5872,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6652408838272095,
+      "rewards/margins": 0.7498928308486938,
+      "rewards/rejected": -2.4151339530944824,
+      "step": 1920
+    },
+    {
+      "epoch": 0.28411600176652435,
+      "grad_norm": 78.5464549732301,
+      "learning_rate": 4.501650544260848e-07,
+      "logits/chosen": -1.5278265476226807,
+      "logits/rejected": -1.6430070400238037,
+      "logps/chosen": -345.4292297363281,
+      "logps/rejected": -400.3189697265625,
+      "loss": 0.5005,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1476633548736572,
+      "rewards/margins": 0.5958186388015747,
+      "rewards/rejected": -1.743481993675232,
+      "step": 1930
+    },
+    {
+      "epoch": 0.28558810540262036,
+      "grad_norm": 91.29964298945447,
+      "learning_rate": 4.4939266687372636e-07,
+      "logits/chosen": -1.608768105506897,
+      "logits/rejected": -1.5200564861297607,
+      "logps/chosen": -511.80303955078125,
+      "logps/rejected": -502.6783142089844,
+      "loss": 0.5877,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -1.6708977222442627,
+      "rewards/margins": 0.807176411151886,
+      "rewards/rejected": -2.478074312210083,
+      "step": 1940
+    },
+    {
+      "epoch": 0.2870602090387163,
+      "grad_norm": 146.2232993455667,
+      "learning_rate": 4.4861501309916185e-07,
+      "logits/chosen": -1.72658371925354,
+      "logits/rejected": -1.5450631380081177,
+      "logps/chosen": -422.600341796875,
+      "logps/rejected": -451.49737548828125,
+      "loss": 0.4793,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.633721113204956,
+      "rewards/margins": 1.042769193649292,
+      "rewards/rejected": -2.676490306854248,
+      "step": 1950
+    },
+    {
+      "epoch": 0.28853231267481233,
+      "grad_norm": 73.58047268290248,
+      "learning_rate": 4.478321136412487e-07,
+      "logits/chosen": -1.400095820426941,
+      "logits/rejected": -1.4668046236038208,
+      "logps/chosen": -502.67254638671875,
+      "logps/rejected": -658.44140625,
+      "loss": 0.6173,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.381251811981201,
+      "rewards/margins": 1.2020304203033447,
+      "rewards/rejected": -3.583281993865967,
+      "step": 1960
+    },
+    {
+      "epoch": 0.2900044163109083,
+      "grad_norm": 45.42567695202515,
+      "learning_rate": 4.4704398917738996e-07,
+      "logits/chosen": -1.6204618215560913,
+      "logits/rejected": -1.6879221200942993,
+      "logps/chosen": -524.9888305664062,
+      "logps/rejected": -639.3353271484375,
+      "loss": 0.5938,
+      "rewards/accuracies": 0.5666666626930237,
+      "rewards/chosen": -2.5351343154907227,
+      "rewards/margins": 0.9098974466323853,
+      "rewards/rejected": -3.4450316429138184,
+      "step": 1970
+    },
+    {
+      "epoch": 0.29147651994700424,
+      "grad_norm": 71.48299353606512,
+      "learning_rate": 4.4625066052298766e-07,
+      "logits/chosen": -1.615454912185669,
+      "logits/rejected": -1.6648781299591064,
+      "logps/chosen": -456.6764221191406,
+      "logps/rejected": -564.7952880859375,
+      "loss": 0.6478,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -2.189084768295288,
+      "rewards/margins": 0.40208807587623596,
+      "rewards/rejected": -2.591172933578491,
+      "step": 1980
+    },
+    {
+      "epoch": 0.29294862358310025,
+      "grad_norm": 45.538931937805685,
+      "learning_rate": 4.454521486308936e-07,
+      "logits/chosen": -1.8534252643585205,
+      "logits/rejected": -1.7413625717163086,
+      "logps/chosen": -533.1121215820312,
+      "logps/rejected": -546.1458129882812,
+      "loss": 0.5436,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -2.2162203788757324,
+      "rewards/margins": 0.5562283396720886,
+      "rewards/rejected": -2.772448778152466,
+      "step": 1990
+    },
+    {
+      "epoch": 0.2944207272191962,
+      "grad_norm": 76.26497187556322,
+      "learning_rate": 4.4464847459085553e-07,
+      "logits/chosen": -1.7512187957763672,
+      "logits/rejected": -1.7058212757110596,
+      "logps/chosen": -466.07733154296875,
+      "logps/rejected": -524.7542724609375,
+      "loss": 0.4741,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.9114131927490234,
+      "rewards/margins": 0.738069474697113,
+      "rewards/rejected": -2.6494827270507812,
+      "step": 2000
+    },
+    {
+      "epoch": 0.2958928308552922,
+      "grad_norm": 98.0863216532731,
+      "learning_rate": 4.438396596289604e-07,
+      "logits/chosen": -1.6177749633789062,
+      "logits/rejected": -1.5478427410125732,
+      "logps/chosen": -444.4658203125,
+      "logps/rejected": -530.3812255859375,
+      "loss": 0.6137,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.7594554424285889,
+      "rewards/margins": 1.1266467571258545,
+      "rewards/rejected": -2.8861021995544434,
+      "step": 2010
+    },
+    {
+      "epoch": 0.2973649344913882,
+      "grad_norm": 44.185656245031204,
+      "learning_rate": 4.430257251070736e-07,
+      "logits/chosen": -1.6525189876556396,
+      "logits/rejected": -1.64784836769104,
+      "logps/chosen": -414.99462890625,
+      "logps/rejected": -457.1453552246094,
+      "loss": 0.5189,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.8041388988494873,
+      "rewards/margins": 0.7363736629486084,
+      "rewards/rejected": -2.5405125617980957,
+      "step": 2020
+    },
+    {
+      "epoch": 0.2988370381274842,
+      "grad_norm": 105.60349488334649,
+      "learning_rate": 4.422066925222748e-07,
+      "logits/chosen": -1.412907600402832,
+      "logits/rejected": -1.3582699298858643,
+      "logps/chosen": -414.4461975097656,
+      "logps/rejected": -481.41436767578125,
+      "loss": 0.5359,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.7802703380584717,
+      "rewards/margins": 0.8613284826278687,
+      "rewards/rejected": -2.641598701477051,
+      "step": 2030
+    },
+    {
+      "epoch": 0.30030914176358015,
+      "grad_norm": 72.58853766247252,
+      "learning_rate": 4.4138258350629033e-07,
+      "logits/chosen": -1.8623470067977905,
+      "logits/rejected": -1.5975165367126465,
+      "logps/chosen": -466.7835388183594,
+      "logps/rejected": -492.2386779785156,
+      "loss": 0.5532,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.3512439727783203,
+      "rewards/margins": 0.8019021153450012,
+      "rewards/rejected": -2.153146266937256,
+      "step": 2040
+    },
+    {
+      "epoch": 0.30178124539967616,
+      "grad_norm": 90.60349334079292,
+      "learning_rate": 4.405534198249216e-07,
+      "logits/chosen": -1.810738205909729,
+      "logits/rejected": -1.5894935131072998,
+      "logps/chosen": -369.27734375,
+      "logps/rejected": -417.78399658203125,
+      "loss": 0.513,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.2387871742248535,
+      "rewards/margins": 0.8931953310966492,
+      "rewards/rejected": -2.1319825649261475,
+      "step": 2050
+    },
+    {
+      "epoch": 0.3032533490357721,
+      "grad_norm": 145.52787685203137,
+      "learning_rate": 4.3971922337747045e-07,
+      "logits/chosen": -1.7671382427215576,
+      "logits/rejected": -1.7577235698699951,
+      "logps/chosen": -387.68212890625,
+      "logps/rejected": -431.3033142089844,
+      "loss": 0.4633,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.1763545274734497,
+      "rewards/margins": 0.864321231842041,
+      "rewards/rejected": -2.0406758785247803,
+      "step": 2060
+    },
+    {
+      "epoch": 0.3047254526718681,
+      "grad_norm": 128.78691357641787,
+      "learning_rate": 4.388800161961606e-07,
+      "logits/chosen": -1.8475401401519775,
+      "logits/rejected": -1.5575668811798096,
+      "logps/chosen": -488.7298278808594,
+      "logps/rejected": -542.2572631835938,
+      "loss": 0.4952,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.9894729852676392,
+      "rewards/margins": 1.0202587842941284,
+      "rewards/rejected": -3.0097317695617676,
+      "step": 2070
+    },
+    {
+      "epoch": 0.3061975563079641,
+      "grad_norm": 100.22977386904307,
+      "learning_rate": 4.380358204455559e-07,
+      "logits/chosen": -1.5467193126678467,
+      "logits/rejected": -1.4157942533493042,
+      "logps/chosen": -437.54541015625,
+      "logps/rejected": -563.5391235351562,
+      "loss": 0.5542,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.265477180480957,
+      "rewards/margins": 0.9674869775772095,
+      "rewards/rejected": -3.232964038848877,
+      "step": 2080
+    },
+    {
+      "epoch": 0.30766965994406004,
+      "grad_norm": 136.05047237428258,
+      "learning_rate": 4.3718665842197494e-07,
+      "logits/chosen": -1.4792824983596802,
+      "logits/rejected": -1.2502820491790771,
+      "logps/chosen": -435.041015625,
+      "logps/rejected": -483.5919494628906,
+      "loss": 0.5162,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1088335514068604,
+      "rewards/margins": 0.9096527099609375,
+      "rewards/rejected": -3.018486499786377,
+      "step": 2090
+    },
+    {
+      "epoch": 0.30914176358015605,
+      "grad_norm": 90.16904640691057,
+      "learning_rate": 4.363325525529019e-07,
+      "logits/chosen": -1.6123536825180054,
+      "logits/rejected": -1.5360729694366455,
+      "logps/chosen": -494.77874755859375,
+      "logps/rejected": -537.6286010742188,
+      "loss": 0.5353,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.2117531299591064,
+      "rewards/margins": 0.9112815856933594,
+      "rewards/rejected": -3.123034954071045,
+      "step": 2100
+    },
+    {
+      "epoch": 0.310613867216252,
+      "grad_norm": 53.0978697251245,
+      "learning_rate": 4.354735253963947e-07,
+      "logits/chosen": -1.6993602514266968,
+      "logits/rejected": -1.6057443618774414,
+      "logps/chosen": -404.397216796875,
+      "logps/rejected": -487.01544189453125,
+      "loss": 0.5217,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.6291040182113647,
+      "rewards/margins": 1.011840581893921,
+      "rewards/rejected": -2.640944480895996,
+      "step": 2110
+    },
+    {
+      "epoch": 0.312085970852348,
+      "grad_norm": 102.9771705233389,
+      "learning_rate": 4.3460959964048854e-07,
+      "logits/chosen": -1.8225218057632446,
+      "logits/rejected": -1.703957200050354,
+      "logps/chosen": -503.252197265625,
+      "logps/rejected": -531.5034790039062,
+      "loss": 0.6341,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -2.0066771507263184,
+      "rewards/margins": 0.5090205073356628,
+      "rewards/rejected": -2.515697717666626,
+      "step": 2120
+    },
+    {
+      "epoch": 0.31355807448844397,
+      "grad_norm": 101.6792683943578,
+      "learning_rate": 4.337407981025974e-07,
+      "logits/chosen": -1.9180629253387451,
+      "logits/rejected": -1.729644536972046,
+      "logps/chosen": -505.4967346191406,
+      "logps/rejected": -643.906005859375,
+      "loss": 0.5094,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.9095513820648193,
+      "rewards/margins": 1.203673005104065,
+      "rewards/rejected": -3.113224506378174,
+      "step": 2130
+    },
+    {
+      "epoch": 0.31503017812454,
+      "grad_norm": 80.68076892749208,
+      "learning_rate": 4.3286714372891086e-07,
+      "logits/chosen": -1.854379415512085,
+      "logits/rejected": -1.736157774925232,
+      "logps/chosen": -399.6598815917969,
+      "logps/rejected": -480.1727600097656,
+      "loss": 0.5159,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.61972177028656,
+      "rewards/margins": 0.7012790441513062,
+      "rewards/rejected": -2.321000814437866,
+      "step": 2140
+    },
+    {
+      "epoch": 0.31650228176063594,
+      "grad_norm": 176.42924773487638,
+      "learning_rate": 4.319886595937885e-07,
+      "logits/chosen": -1.83878493309021,
+      "logits/rejected": -1.7423222064971924,
+      "logps/chosen": -476.2215881347656,
+      "logps/rejected": -619.8114013671875,
+      "loss": 0.3495,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -1.5030966997146606,
+      "rewards/margins": 1.5744469165802002,
+      "rewards/rejected": -3.0775437355041504,
+      "step": 2150
+    },
+    {
+      "epoch": 0.31797438539673195,
+      "grad_norm": 113.200267427989,
+      "learning_rate": 4.3110536889914996e-07,
+      "logits/chosen": -1.9114919900894165,
+      "logits/rejected": -1.7372190952301025,
+      "logps/chosen": -460.77178955078125,
+      "logps/rejected": -538.3016357421875,
+      "loss": 0.4228,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.6960121393203735,
+      "rewards/margins": 1.0352016687393188,
+      "rewards/rejected": -2.7312138080596924,
+      "step": 2160
+    },
+    {
+      "epoch": 0.3194464890328279,
+      "grad_norm": 168.59281094878145,
+      "learning_rate": 4.302172949738626e-07,
+      "logits/chosen": -1.5778995752334595,
+      "logits/rejected": -1.4069231748580933,
+      "logps/chosen": -462.450927734375,
+      "logps/rejected": -779.4312744140625,
+      "loss": 0.3524,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.3175973892211914,
+      "rewards/margins": 2.8890976905822754,
+      "rewards/rejected": -5.206694602966309,
+      "step": 2170
+    },
+    {
+      "epoch": 0.3209185926689239,
+      "grad_norm": 278.69400660565157,
+      "learning_rate": 4.2932446127312516e-07,
+      "logits/chosen": -1.5741561651229858,
+      "logits/rejected": -1.5650957822799683,
+      "logps/chosen": -478.7816467285156,
+      "logps/rejected": -707.4798583984375,
+      "loss": 0.5197,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.383969306945801,
+      "rewards/margins": 1.699548363685608,
+      "rewards/rejected": -4.083518028259277,
+      "step": 2180
+    },
+    {
+      "epoch": 0.3223906963050199,
+      "grad_norm": 173.69754700155016,
+      "learning_rate": 4.2842689137784825e-07,
+      "logits/chosen": -1.6903053522109985,
+      "logits/rejected": -1.4653757810592651,
+      "logps/chosen": -617.8346557617188,
+      "logps/rejected": -616.084228515625,
+      "loss": 0.6035,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -3.0368387699127197,
+      "rewards/margins": 0.8624863624572754,
+      "rewards/rejected": -3.899325132369995,
+      "step": 2190
+    },
+    {
+      "epoch": 0.32386279994111583,
+      "grad_norm": 46.58029651401236,
+      "learning_rate": 4.2752460899403175e-07,
+      "logits/chosen": -1.7009000778198242,
+      "logits/rejected": -1.5140489339828491,
+      "logps/chosen": -391.1360778808594,
+      "logps/rejected": -509.20233154296875,
+      "loss": 0.5781,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.7962887287139893,
+      "rewards/margins": 0.9368594884872437,
+      "rewards/rejected": -2.7331480979919434,
+      "step": 2200
+    },
+    {
+      "epoch": 0.32533490357721184,
+      "grad_norm": 81.66337340445386,
+      "learning_rate": 4.2661763795213824e-07,
+      "logits/chosen": -1.603149175643921,
+      "logits/rejected": -1.5082519054412842,
+      "logps/chosen": -343.338623046875,
+      "logps/rejected": -503.3509826660156,
+      "loss": 0.4829,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -1.1309901475906372,
+      "rewards/margins": 1.4072386026382446,
+      "rewards/rejected": -2.538228750228882,
+      "step": 2210
+    },
+    {
+      "epoch": 0.3268070072133078,
+      "grad_norm": 99.90977774610141,
+      "learning_rate": 4.2570600220646425e-07,
+      "logits/chosen": -1.8064130544662476,
+      "logits/rejected": -1.7388206720352173,
+      "logps/chosen": -425.31982421875,
+      "logps/rejected": -491.24993896484375,
+      "loss": 0.5276,
+      "rewards/accuracies": 0.5999999642372131,
+      "rewards/chosen": -1.5650538206100464,
+      "rewards/margins": 0.8042703866958618,
+      "rewards/rejected": -2.369324207305908,
+      "step": 2220
+    },
+    {
+      "epoch": 0.3282791108494038,
+      "grad_norm": 115.65571685368333,
+      "learning_rate": 4.247897258345071e-07,
+      "logits/chosen": -1.7937599420547485,
+      "logits/rejected": -1.548876166343689,
+      "logps/chosen": -518.07373046875,
+      "logps/rejected": -522.1268310546875,
+      "loss": 0.4908,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.983019232749939,
+      "rewards/margins": 0.8229973912239075,
+      "rewards/rejected": -2.806016445159912,
+      "step": 2230
+    },
+    {
+      "epoch": 0.32975121448549977,
+      "grad_norm": 276.4722022286063,
+      "learning_rate": 4.238688330363292e-07,
+      "logits/chosen": -1.5255985260009766,
+      "logits/rejected": -1.411313772201538,
+      "logps/chosen": -432.21160888671875,
+      "logps/rejected": -520.4403076171875,
+      "loss": 0.5167,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.198775053024292,
+      "rewards/margins": 1.0241445302963257,
+      "rewards/rejected": -3.222919464111328,
+      "step": 2240
+    },
+    {
+      "epoch": 0.3312233181215958,
+      "grad_norm": 108.0234973910894,
+      "learning_rate": 4.2294334813391874e-07,
+      "logits/chosen": -1.619693398475647,
+      "logits/rejected": -1.6066100597381592,
+      "logps/chosen": -473.578125,
+      "logps/rejected": -597.0567016601562,
+      "loss": 0.4687,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.8159301280975342,
+      "rewards/margins": 0.8482751846313477,
+      "rewards/rejected": -2.66420578956604,
+      "step": 2250
+    },
+    {
+      "epoch": 0.33269542175769173,
+      "grad_norm": 201.23427831937016,
+      "learning_rate": 4.220132955705476e-07,
+      "logits/chosen": -1.6948457956314087,
+      "logits/rejected": -1.591141939163208,
+      "logps/chosen": -404.19439697265625,
+      "logps/rejected": -464.029052734375,
+      "loss": 0.6185,
+      "rewards/accuracies": 0.7333332300186157,
+      "rewards/chosen": -1.5307927131652832,
+      "rewards/margins": 0.9377376437187195,
+      "rewards/rejected": -2.4685301780700684,
+      "step": 2260
+    },
+    {
+      "epoch": 0.33416752539378775,
+      "grad_norm": 70.83276045446581,
+      "learning_rate": 4.2107869991012536e-07,
+      "logits/chosen": -1.7897859811782837,
+      "logits/rejected": -1.5137708187103271,
+      "logps/chosen": -467.86151123046875,
+      "logps/rejected": -523.0823974609375,
+      "loss": 0.4469,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.504129409790039,
+      "rewards/margins": 1.249953031539917,
+      "rewards/rejected": -2.754082441329956,
+      "step": 2270
+    },
+    {
+      "epoch": 0.3356396290298837,
+      "grad_norm": 83.36314620650249,
+      "learning_rate": 4.201395858365509e-07,
+      "logits/chosen": -1.6233571767807007,
+      "logits/rejected": -1.6392791271209717,
+      "logps/chosen": -397.0567626953125,
+      "logps/rejected": -519.8914794921875,
+      "loss": 0.5112,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -1.665471076965332,
+      "rewards/margins": 0.7333282232284546,
+      "rewards/rejected": -2.398799419403076,
+      "step": 2280
+    },
+    {
+      "epoch": 0.33711173266597966,
+      "grad_norm": 90.54641182859237,
+      "learning_rate": 4.191959781530603e-07,
+      "logits/chosen": -1.74801766872406,
+      "logits/rejected": -1.6157410144805908,
+      "logps/chosen": -396.1054382324219,
+      "logps/rejected": -462.4190979003906,
+      "loss": 0.4772,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -1.3713312149047852,
+      "rewards/margins": 1.041359782218933,
+      "rewards/rejected": -2.4126908779144287,
+      "step": 2290
+    },
+    {
+      "epoch": 0.33858383630207567,
+      "grad_norm": 275.75693491109797,
+      "learning_rate": 4.1824790178157184e-07,
+      "logits/chosen": -1.643858551979065,
+      "logits/rejected": -1.4274442195892334,
+      "logps/chosen": -501.07159423828125,
+      "logps/rejected": -533.362548828125,
+      "loss": 0.5849,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.1280364990234375,
+      "rewards/margins": 0.8686404228210449,
+      "rewards/rejected": -2.9966769218444824,
+      "step": 2300
+    },
+    {
+      "epoch": 0.3400559399381716,
+      "grad_norm": 133.68453914825938,
+      "learning_rate": 4.172953817620275e-07,
+      "logits/chosen": -1.5242453813552856,
+      "logits/rejected": -1.4907231330871582,
+      "logps/chosen": -424.06866455078125,
+      "logps/rejected": -493.7257385253906,
+      "loss": 0.6211,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9947437047958374,
+      "rewards/margins": 0.778528094291687,
+      "rewards/rejected": -2.7732717990875244,
+      "step": 2310
+    },
+    {
+      "epoch": 0.34152804357426764,
+      "grad_norm": 88.6513757240061,
+      "learning_rate": 4.1633844325173215e-07,
+      "logits/chosen": -2.0367541313171387,
+      "logits/rejected": -1.789872407913208,
+      "logps/chosen": -447.4058532714844,
+      "logps/rejected": -527.1895751953125,
+      "loss": 0.448,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.5566468238830566,
+      "rewards/margins": 0.866759181022644,
+      "rewards/rejected": -2.4234061241149902,
+      "step": 2320
+    },
+    {
+      "epoch": 0.3430001472103636,
+      "grad_norm": 79.81963281979581,
+      "learning_rate": 4.153771115246886e-07,
+      "logits/chosen": -1.807356595993042,
+      "logits/rejected": -1.7876075506210327,
+      "logps/chosen": -403.1991271972656,
+      "logps/rejected": -438.70452880859375,
+      "loss": 0.5402,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.430990219116211,
+      "rewards/margins": 0.7736988663673401,
+      "rewards/rejected": -2.2046890258789062,
+      "step": 2330
+    },
+    {
+      "epoch": 0.3444722508464596,
+      "grad_norm": 82.9727288780879,
+      "learning_rate": 4.144114119709303e-07,
+      "logits/chosen": -1.8029406070709229,
+      "logits/rejected": -1.7416242361068726,
+      "logps/chosen": -419.9112854003906,
+      "logps/rejected": -491.265625,
+      "loss": 0.4891,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.2236841917037964,
+      "rewards/margins": 1.0209158658981323,
+      "rewards/rejected": -2.244600296020508,
+      "step": 2340
+    },
+    {
+      "epoch": 0.34594435448255556,
+      "grad_norm": 201.68750219209377,
+      "learning_rate": 4.134413700958509e-07,
+      "logits/chosen": -1.7415510416030884,
+      "logits/rejected": -1.6703112125396729,
+      "logps/chosen": -421.6688537597656,
+      "logps/rejected": -517.2791137695312,
+      "loss": 0.4647,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.4775469303131104,
+      "rewards/margins": 1.1024352312088013,
+      "rewards/rejected": -2.579982280731201,
+      "step": 2350
+    },
+    {
+      "epoch": 0.34741645811865157,
+      "grad_norm": 170.76824942269258,
+      "learning_rate": 4.1246701151953014e-07,
+      "logits/chosen": -1.648890495300293,
+      "logits/rejected": -1.5278441905975342,
+      "logps/chosen": -426.95556640625,
+      "logps/rejected": -452.89178466796875,
+      "loss": 0.5879,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.9528697729110718,
+      "rewards/margins": 0.5463830232620239,
+      "rewards/rejected": -2.499253273010254,
+      "step": 2360
+    },
+    {
+      "epoch": 0.34888856175474753,
+      "grad_norm": 152.41564563619943,
+      "learning_rate": 4.1148836197605774e-07,
+      "logits/chosen": -1.845446228981018,
+      "logits/rejected": -1.8086979389190674,
+      "logps/chosen": -420.7265625,
+      "logps/rejected": -522.8754272460938,
+      "loss": 0.5232,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6788885593414307,
+      "rewards/margins": 1.1555010080337524,
+      "rewards/rejected": -2.8343894481658936,
+      "step": 2370
+    },
+    {
+      "epoch": 0.35036066539084354,
+      "grad_norm": 99.5333377976858,
+      "learning_rate": 4.105054473128536e-07,
+      "logits/chosen": -1.5415117740631104,
+      "logits/rejected": -1.571418046951294,
+      "logps/chosen": -339.0941162109375,
+      "logps/rejected": -409.9202575683594,
+      "loss": 0.5038,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.386282205581665,
+      "rewards/margins": 0.786494255065918,
+      "rewards/rejected": -2.172776699066162,
+      "step": 2380
+    },
+    {
+      "epoch": 0.3518327690269395,
+      "grad_norm": 71.6492219061604,
+      "learning_rate": 4.0951829348998477e-07,
+      "logits/chosen": -1.7694034576416016,
+      "logits/rejected": -1.6417080163955688,
+      "logps/chosen": -485.8193359375,
+      "logps/rejected": -523.8450927734375,
+      "loss": 0.5399,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.9029020071029663,
+      "rewards/margins": 0.8837946057319641,
+      "rewards/rejected": -2.7866969108581543,
+      "step": 2390
+    },
+    {
+      "epoch": 0.35330487266303545,
+      "grad_norm": 106.60260065713516,
+      "learning_rate": 4.0852692657948027e-07,
+      "logits/chosen": -1.729047417640686,
+      "logits/rejected": -1.5688316822052002,
+      "logps/chosen": -428.2808532714844,
+      "logps/rejected": -506.84942626953125,
+      "loss": 0.4931,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -2.001753091812134,
+      "rewards/margins": 0.9478896260261536,
+      "rewards/rejected": -2.9496426582336426,
+      "step": 2400
+    },
+    {
+      "epoch": 0.35477697629913146,
+      "grad_norm": 165.64069155776042,
+      "learning_rate": 4.075313727646422e-07,
+      "logits/chosen": -1.7659275531768799,
+      "logits/rejected": -1.659751296043396,
+      "logps/chosen": -427.1341857910156,
+      "logps/rejected": -658.7684326171875,
+      "loss": 0.4037,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -1.53098464012146,
+      "rewards/margins": 2.267085075378418,
+      "rewards/rejected": -3.798069715499878,
+      "step": 2410
+    },
+    {
+      "epoch": 0.3562490799352274,
+      "grad_norm": 80.33914214981517,
+      "learning_rate": 4.0653165833935433e-07,
+      "logits/chosen": -1.7472379207611084,
+      "logits/rejected": -1.6408731937408447,
+      "logps/chosen": -430.6123046875,
+      "logps/rejected": -450.4761657714844,
+      "loss": 0.4999,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.6223323345184326,
+      "rewards/margins": 0.9185541272163391,
+      "rewards/rejected": -2.540886640548706,
+      "step": 2420
+    },
+    {
+      "epoch": 0.35772118357132343,
+      "grad_norm": 155.16471959016783,
+      "learning_rate": 4.0552780970738755e-07,
+      "logits/chosen": -1.6454254388809204,
+      "logits/rejected": -1.722833275794983,
+      "logps/chosen": -406.28668212890625,
+      "logps/rejected": -606.0823364257812,
+      "loss": 0.497,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.685489296913147,
+      "rewards/margins": 1.4034935235977173,
+      "rewards/rejected": -3.0889828205108643,
+      "step": 2430
+    },
+    {
+      "epoch": 0.3591932872074194,
+      "grad_norm": 77.22939282764068,
+      "learning_rate": 4.045198533817028e-07,
+      "logits/chosen": -1.6065928936004639,
+      "logits/rejected": -1.5150026082992554,
+      "logps/chosen": -460.1014709472656,
+      "logps/rejected": -479.58551025390625,
+      "loss": 0.5253,
+      "rewards/accuracies": 0.5666666626930237,
+      "rewards/chosen": -2.1393537521362305,
+      "rewards/margins": 0.4858306050300598,
+      "rewards/rejected": -2.6251842975616455,
+      "step": 2440
+    },
+    {
+      "epoch": 0.3606653908435154,
+      "grad_norm": 72.37032769518197,
+      "learning_rate": 4.0350781598375027e-07,
+      "logits/chosen": -1.8797813653945923,
+      "logits/rejected": -1.5134384632110596,
+      "logps/chosen": -555.326416015625,
+      "logps/rejected": -551.3318481445312,
+      "loss": 0.5071,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8028364181518555,
+      "rewards/margins": 1.157103180885315,
+      "rewards/rejected": -2.95993971824646,
+      "step": 2450
+    },
+    {
+      "epoch": 0.36213749447961135,
+      "grad_norm": 44.34521151604621,
+      "learning_rate": 4.024917242427669e-07,
+      "logits/chosen": -1.761695146560669,
+      "logits/rejected": -1.6549463272094727,
+      "logps/chosen": -493.4774475097656,
+      "logps/rejected": -565.1205444335938,
+      "loss": 0.5217,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -2.094701051712036,
+      "rewards/margins": 0.9296547770500183,
+      "rewards/rejected": -3.02435564994812,
+      "step": 2460
+    },
+    {
+      "epoch": 0.36360959811570737,
+      "grad_norm": 131.94375909507465,
+      "learning_rate": 4.0147160499507006e-07,
+      "logits/chosen": -1.7216295003890991,
+      "logits/rejected": -1.5743701457977295,
+      "logps/chosen": -556.4461059570312,
+      "logps/rejected": -554.2019653320312,
+      "loss": 0.5745,
+      "rewards/accuracies": 0.5666666626930237,
+      "rewards/chosen": -2.2372331619262695,
+      "rewards/margins": 0.646490752696991,
+      "rewards/rejected": -2.883723735809326,
+      "step": 2470
+    },
+    {
+      "epoch": 0.3650817017518033,
+      "grad_norm": 86.02155564362292,
+      "learning_rate": 4.004474851833488e-07,
+      "logits/chosen": -1.9678194522857666,
+      "logits/rejected": -1.798370361328125,
+      "logps/chosen": -569.799560546875,
+      "logps/rejected": -543.369140625,
+      "loss": 0.4965,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6525856256484985,
+      "rewards/margins": 0.9285826683044434,
+      "rewards/rejected": -2.5811684131622314,
+      "step": 2480
+    },
+    {
+      "epoch": 0.36655380538789933,
+      "grad_norm": 69.154632954316,
+      "learning_rate": 3.9941939185595255e-07,
+      "logits/chosen": -1.6970447301864624,
+      "logits/rejected": -1.6858055591583252,
+      "logps/chosen": -420.1490783691406,
+      "logps/rejected": -516.6390380859375,
+      "loss": 0.5071,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.7808294296264648,
+      "rewards/margins": 0.8183453679084778,
+      "rewards/rejected": -2.599174976348877,
+      "step": 2490
+    },
+    {
+      "epoch": 0.3680259090239953,
+      "grad_norm": 51.65539198848958,
+      "learning_rate": 3.9838735216617615e-07,
+      "logits/chosen": -1.9549919366836548,
+      "logits/rejected": -1.7374999523162842,
+      "logps/chosen": -404.2395324707031,
+      "logps/rejected": -411.1822204589844,
+      "loss": 0.5923,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.5497435331344604,
+      "rewards/margins": 0.46222543716430664,
+      "rewards/rejected": -2.0119690895080566,
+      "step": 2500
+    },
+    {
+      "epoch": 0.36949801266009125,
+      "grad_norm": 108.0607252501813,
+      "learning_rate": 3.9735139337154334e-07,
+      "logits/chosen": -1.6869624853134155,
+      "logits/rejected": -1.535576343536377,
+      "logps/chosen": -372.4644775390625,
+      "logps/rejected": -396.6224365234375,
+      "loss": 0.7073,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.7658885717391968,
+      "rewards/margins": 0.41729116439819336,
+      "rewards/rejected": -2.1831798553466797,
+      "step": 2510
+    },
+    {
+      "epoch": 0.37097011629618726,
+      "grad_norm": 205.528312081728,
+      "learning_rate": 3.963115428330864e-07,
+      "logits/chosen": -1.9358739852905273,
+      "logits/rejected": -1.8595073223114014,
+      "logps/chosen": -477.067138671875,
+      "logps/rejected": -510.2032165527344,
+      "loss": 0.5285,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.340234398841858,
+      "rewards/margins": 0.4346332550048828,
+      "rewards/rejected": -1.7748676538467407,
+      "step": 2520
+    },
+    {
+      "epoch": 0.3724422199322832,
+      "grad_norm": 62.93162090010597,
+      "learning_rate": 3.9526782801462384e-07,
+      "logits/chosen": -1.9853556156158447,
+      "logits/rejected": -1.5232021808624268,
+      "logps/chosen": -439.63507080078125,
+      "logps/rejected": -455.38641357421875,
+      "loss": 0.4636,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -1.28523850440979,
+      "rewards/margins": 1.3732984066009521,
+      "rewards/rejected": -2.658536672592163,
+      "step": 2530
+    },
+    {
+      "epoch": 0.3739143235683792,
+      "grad_norm": 92.80607468315416,
+      "learning_rate": 3.9422027648203474e-07,
+      "logits/chosen": -1.4608604907989502,
+      "logits/rejected": -1.4058932065963745,
+      "logps/chosen": -407.0013122558594,
+      "logps/rejected": -442.52569580078125,
+      "loss": 0.5356,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.1296303272247314,
+      "rewards/margins": 0.35629209876060486,
+      "rewards/rejected": -2.485922336578369,
+      "step": 2540
+    },
+    {
+      "epoch": 0.3753864272044752,
+      "grad_norm": 79.96693950549795,
+      "learning_rate": 3.9316891590253086e-07,
+      "logits/chosen": -1.9654786586761475,
+      "logits/rejected": -1.748862862586975,
+      "logps/chosen": -483.31915283203125,
+      "logps/rejected": -529.26025390625,
+      "loss": 0.5462,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.003206729888916,
+      "rewards/margins": 0.8893306851387024,
+      "rewards/rejected": -2.8925373554229736,
+      "step": 2550
+    },
+    {
+      "epoch": 0.3768585308405712,
+      "grad_norm": 56.42140357987996,
+      "learning_rate": 3.921137740439261e-07,
+      "logits/chosen": -1.9938732385635376,
+      "logits/rejected": -1.8925504684448242,
+      "logps/chosen": -409.49664306640625,
+      "logps/rejected": -457.82708740234375,
+      "loss": 0.4268,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.618497610092163,
+      "rewards/margins": 0.8208115696907043,
+      "rewards/rejected": -2.4393093585968018,
+      "step": 2560
+    },
+    {
+      "epoch": 0.37833063447666715,
+      "grad_norm": 66.24397541381366,
+      "learning_rate": 3.910548787739024e-07,
+      "logits/chosen": -2.0649988651275635,
+      "logits/rejected": -1.8993663787841797,
+      "logps/chosen": -502.3565368652344,
+      "logps/rejected": -589.432373046875,
+      "loss": 0.5242,
+      "rewards/accuracies": 0.73333340883255,
+      "rewards/chosen": -2.220254421234131,
+      "rewards/margins": 0.9584907293319702,
+      "rewards/rejected": -3.1787452697753906,
+      "step": 2570
+    },
+    {
+      "epoch": 0.37980273811276316,
+      "grad_norm": 88.89202049332125,
+      "learning_rate": 3.8999225805927483e-07,
+      "logits/chosen": -1.9994415044784546,
+      "logits/rejected": -1.6600345373153687,
+      "logps/chosen": -461.4424743652344,
+      "logps/rejected": -531.6176147460938,
+      "loss": 0.5575,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -2.0529115200042725,
+      "rewards/margins": 0.8937948942184448,
+      "rewards/rejected": -2.9467062950134277,
+      "step": 2580
+    },
+    {
+      "epoch": 0.3812748417488591,
+      "grad_norm": 57.87439072258639,
+      "learning_rate": 3.889259399652519e-07,
+      "logits/chosen": -1.8909122943878174,
+      "logits/rejected": -1.6752264499664307,
+      "logps/chosen": -532.424560546875,
+      "logps/rejected": -600.0192260742188,
+      "loss": 0.4089,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -2.1988584995269775,
+      "rewards/margins": 1.2118005752563477,
+      "rewards/rejected": -3.410658597946167,
+      "step": 2590
+    },
+    {
+      "epoch": 0.38274694538495513,
+      "grad_norm": 187.69946919712814,
+      "learning_rate": 3.8785595265469497e-07,
+      "logits/chosen": -1.657313346862793,
+      "logits/rejected": -1.617428183555603,
+      "logps/chosen": -478.59722900390625,
+      "logps/rejected": -561.6739501953125,
+      "loss": 0.4847,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.2130188941955566,
+      "rewards/margins": 1.131255865097046,
+      "rewards/rejected": -3.3442745208740234,
+      "step": 2600
+    },
+    {
+      "epoch": 0.3842190490210511,
+      "grad_norm": 126.62049634548895,
+      "learning_rate": 3.867823243873743e-07,
+      "logits/chosen": -1.6318082809448242,
+      "logits/rejected": -1.4904166460037231,
+      "logps/chosen": -422.03607177734375,
+      "logps/rejected": -512.8194580078125,
+      "loss": 0.414,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -2.04823637008667,
+      "rewards/margins": 1.1126116514205933,
+      "rewards/rejected": -3.1608481407165527,
+      "step": 2610
+    },
+    {
+      "epoch": 0.38569115265714704,
+      "grad_norm": 84.95647857185706,
+      "learning_rate": 3.8570508351922234e-07,
+      "logits/chosen": -1.6178743839263916,
+      "logits/rejected": -1.45248544216156,
+      "logps/chosen": -510.5176696777344,
+      "logps/rejected": -627.6212158203125,
+      "loss": 0.4889,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.243734836578369,
+      "rewards/margins": 1.4953174591064453,
+      "rewards/rejected": -3.7390522956848145,
+      "step": 2620
+    },
+    {
+      "epoch": 0.38716325629324305,
+      "grad_norm": 217.3610849910934,
+      "learning_rate": 3.8462425850158533e-07,
+      "logits/chosen": -1.5772051811218262,
+      "logits/rejected": -1.6962175369262695,
+      "logps/chosen": -454.8011169433594,
+      "logps/rejected": -596.3443603515625,
+      "loss": 0.5157,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -2.3009824752807617,
+      "rewards/margins": 1.1165456771850586,
+      "rewards/rejected": -3.4175286293029785,
+      "step": 2630
+    },
+    {
+      "epoch": 0.388635359929339,
+      "grad_norm": 93.71271391232355,
+      "learning_rate": 3.8353987788047136e-07,
+      "logits/chosen": -1.7966476678848267,
+      "logits/rejected": -1.6472831964492798,
+      "logps/chosen": -525.8533935546875,
+      "logps/rejected": -578.4793090820312,
+      "loss": 0.4526,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -2.2422289848327637,
+      "rewards/margins": 0.9489534497261047,
+      "rewards/rejected": -3.1911826133728027,
+      "step": 2640
+    },
+    {
+      "epoch": 0.390107463565435,
+      "grad_norm": 117.14755197392523,
+      "learning_rate": 3.824519702957969e-07,
+      "logits/chosen": -1.8352773189544678,
+      "logits/rejected": -1.7037101984024048,
+      "logps/chosen": -601.3973999023438,
+      "logps/rejected": -587.599609375,
+      "loss": 0.7123,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -2.5357017517089844,
+      "rewards/margins": 0.8538087606430054,
+      "rewards/rejected": -3.3895103931427,
+      "step": 2650
+    },
+    {
+      "epoch": 0.391579567201531,
+      "grad_norm": 100.67283128846917,
+      "learning_rate": 3.8136056448063016e-07,
+      "logits/chosen": -1.969740867614746,
+      "logits/rejected": -1.8629324436187744,
+      "logps/chosen": -484.20343017578125,
+      "logps/rejected": -623.115966796875,
+      "loss": 0.5587,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -2.134702682495117,
+      "rewards/margins": 1.5543760061264038,
+      "rewards/rejected": -3.6890788078308105,
+      "step": 2660
+    },
+    {
+      "epoch": 0.393051670837627,
+      "grad_norm": 68.67861913584042,
+      "learning_rate": 3.802656892604319e-07,
+      "logits/chosen": -1.7493486404418945,
+      "logits/rejected": -1.6327745914459229,
+      "logps/chosen": -416.34039306640625,
+      "logps/rejected": -526.93603515625,
+      "loss": 0.5177,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.926028847694397,
+      "rewards/margins": 1.4044030904769897,
+      "rewards/rejected": -3.330432176589966,
+      "step": 2670
+    },
+    {
+      "epoch": 0.39452377447372294,
+      "grad_norm": 83.42034644974696,
+      "learning_rate": 3.791673735522949e-07,
+      "logits/chosen": -1.7801506519317627,
+      "logits/rejected": -1.6689989566802979,
+      "logps/chosen": -450.91168212890625,
+      "logps/rejected": -604.1636962890625,
+      "loss": 0.5419,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.969585657119751,
+      "rewards/margins": 1.1824830770492554,
+      "rewards/rejected": -3.152069091796875,
+      "step": 2680
+    },
+    {
+      "epoch": 0.39599587810981896,
+      "grad_norm": 92.00474092190898,
+      "learning_rate": 3.7806564636417936e-07,
+      "logits/chosen": -1.8807475566864014,
+      "logits/rejected": -1.7490568161010742,
+      "logps/chosen": -384.54510498046875,
+      "logps/rejected": -480.938232421875,
+      "loss": 0.5335,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -1.2384569644927979,
+      "rewards/margins": 1.1380057334899902,
+      "rewards/rejected": -2.376462936401367,
+      "step": 2690
+    },
+    {
+      "epoch": 0.3974679817459149,
+      "grad_norm": 61.0433259916579,
+      "learning_rate": 3.769605367941472e-07,
+      "logits/chosen": -1.7871767282485962,
+      "logits/rejected": -1.7239713668823242,
+      "logps/chosen": -387.7853698730469,
+      "logps/rejected": -457.2977600097656,
+      "loss": 0.43,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.350097894668579,
+      "rewards/margins": 0.8648613095283508,
+      "rewards/rejected": -2.214958906173706,
+      "step": 2700
+    },
+    {
+      "epoch": 0.39894008538201087,
+      "grad_norm": 91.02215737867614,
+      "learning_rate": 3.7585207402959377e-07,
+      "logits/chosen": -1.882582426071167,
+      "logits/rejected": -1.7801997661590576,
+      "logps/chosen": -335.1067199707031,
+      "logps/rejected": -406.0661315917969,
+      "loss": 0.5354,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -0.9968665838241577,
+      "rewards/margins": 0.7382841110229492,
+      "rewards/rejected": -1.735150694847107,
+      "step": 2710
+    },
+    {
+      "epoch": 0.4004121890181069,
+      "grad_norm": 39.49209571031024,
+      "learning_rate": 3.747402873464764e-07,
+      "logits/chosen": -1.8460311889648438,
+      "logits/rejected": -1.5826148986816406,
+      "logps/chosen": -379.17303466796875,
+      "logps/rejected": -449.3412170410156,
+      "loss": 0.4803,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.0963877439498901,
+      "rewards/margins": 1.0909336805343628,
+      "rewards/rejected": -2.187321424484253,
+      "step": 2720
+    },
+    {
+      "epoch": 0.40188429265420283,
+      "grad_norm": 499.5492917203205,
+      "learning_rate": 3.7362520610854147e-07,
+      "logits/chosen": -1.8000824451446533,
+      "logits/rejected": -1.6837491989135742,
+      "logps/chosen": -302.5549621582031,
+      "logps/rejected": -401.45684814453125,
+      "loss": 0.539,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -0.9969806671142578,
+      "rewards/margins": 0.9101873636245728,
+      "rewards/rejected": -1.9071677923202515,
+      "step": 2730
+    },
+    {
+      "epoch": 0.40335639629029885,
+      "grad_norm": 189.95202996672072,
+      "learning_rate": 3.725068597665491e-07,
+      "logits/chosen": -1.6278194189071655,
+      "logits/rejected": -1.5598769187927246,
+      "logps/chosen": -362.18670654296875,
+      "logps/rejected": -529.1080322265625,
+      "loss": 0.5633,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -1.668229103088379,
+      "rewards/margins": 0.9493568539619446,
+      "rewards/rejected": -2.617586135864258,
+      "step": 2740
+    },
+    {
+      "epoch": 0.4048284999263948,
+      "grad_norm": 57.17856940250938,
+      "learning_rate": 3.71385277857495e-07,
+      "logits/chosen": -1.5626249313354492,
+      "logits/rejected": -1.5322986841201782,
+      "logps/chosen": -467.0201721191406,
+      "logps/rejected": -592.6168823242188,
+      "loss": 0.6194,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.0915586948394775,
+      "rewards/margins": 1.053040862083435,
+      "rewards/rejected": -3.144599676132202,
+      "step": 2750
+    },
+    {
+      "epoch": 0.4063006035624908,
+      "grad_norm": 123.65770816088971,
+      "learning_rate": 3.702604900038302e-07,
+      "logits/chosen": -1.8136831521987915,
+      "logits/rejected": -1.6411972045898438,
+      "logps/chosen": -401.2938537597656,
+      "logps/rejected": -481.94866943359375,
+      "loss": 0.5359,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.6743227243423462,
+      "rewards/margins": 1.1585530042648315,
+      "rewards/rejected": -2.8328757286071777,
+      "step": 2760
+    },
+    {
+      "epoch": 0.40777270719858677,
+      "grad_norm": 143.9870781335657,
+      "learning_rate": 3.691325259126794e-07,
+      "logits/chosen": -1.6237173080444336,
+      "logits/rejected": -1.6452341079711914,
+      "logps/chosen": -437.26226806640625,
+      "logps/rejected": -552.0536499023438,
+      "loss": 0.5759,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.1095197200775146,
+      "rewards/margins": 0.8526825904846191,
+      "rewards/rejected": -2.962202310562134,
+      "step": 2770
+    },
+    {
+      "epoch": 0.4092448108346828,
+      "grad_norm": 180.00670354693986,
+      "learning_rate": 3.6800141537505556e-07,
+      "logits/chosen": -1.767045259475708,
+      "logits/rejected": -1.6805870532989502,
+      "logps/chosen": -439.8860778808594,
+      "logps/rejected": -606.5364379882812,
+      "loss": 0.4544,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -2.072852373123169,
+      "rewards/margins": 1.1825932264328003,
+      "rewards/rejected": -3.2554454803466797,
+      "step": 2780
+    },
+    {
+      "epoch": 0.41071691447077874,
+      "grad_norm": 200.81887663430837,
+      "learning_rate": 3.6686718826507363e-07,
+      "logits/chosen": -1.7911945581436157,
+      "logits/rejected": -1.6554672718048096,
+      "logps/chosen": -517.91162109375,
+      "logps/rejected": -578.0740356445312,
+      "loss": 0.663,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -2.175565004348755,
+      "rewards/margins": 0.9195197820663452,
+      "rewards/rejected": -3.0950846672058105,
+      "step": 2790
+    },
+    {
+      "epoch": 0.41218901810687475,
+      "grad_norm": 51.678066494413,
+      "learning_rate": 3.6572987453916114e-07,
+      "logits/chosen": -1.5547738075256348,
+      "logits/rejected": -1.453906774520874,
+      "logps/chosen": -441.12518310546875,
+      "logps/rejected": -508.16717529296875,
+      "loss": 0.4146,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.0770745277404785,
+      "rewards/margins": 1.0565335750579834,
+      "rewards/rejected": -3.133607864379883,
+      "step": 2800
+    },
+    {
+      "epoch": 0.4136611217429707,
+      "grad_norm": 100.40668329912171,
+      "learning_rate": 3.645895042352672e-07,
+      "logits/chosen": -1.7121965885162354,
+      "logits/rejected": -1.528241753578186,
+      "logps/chosen": -455.609130859375,
+      "logps/rejected": -613.9754028320312,
+      "loss": 0.4391,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.7937233448028564,
+      "rewards/margins": 1.7376922369003296,
+      "rewards/rejected": -3.5314154624938965,
+      "step": 2810
+    },
+    {
+      "epoch": 0.41513322537906666,
+      "grad_norm": 214.75386932867553,
+      "learning_rate": 3.634461074720695e-07,
+      "logits/chosen": -1.533944845199585,
+      "logits/rejected": -1.544070839881897,
+      "logps/chosen": -434.95330810546875,
+      "logps/rejected": -532.8826904296875,
+      "loss": 0.5883,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9687788486480713,
+      "rewards/margins": 1.018162488937378,
+      "rewards/rejected": -2.986941337585449,
+      "step": 2820
+    },
+    {
+      "epoch": 0.4166053290151627,
+      "grad_norm": 128.46106527602066,
+      "learning_rate": 3.622997144481781e-07,
+      "logits/chosen": -1.6738640069961548,
+      "logits/rejected": -1.3706481456756592,
+      "logps/chosen": -503.4684143066406,
+      "logps/rejected": -481.333740234375,
+      "loss": 0.5221,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.184661388397217,
+      "rewards/margins": 0.5752471089363098,
+      "rewards/rejected": -2.759908437728882,
+      "step": 2830
+    },
+    {
+      "epoch": 0.41807743265125863,
+      "grad_norm": 65.25789469781631,
+      "learning_rate": 3.611503554413383e-07,
+      "logits/chosen": -1.8849303722381592,
+      "logits/rejected": -1.672583818435669,
+      "logps/chosen": -434.37841796875,
+      "logps/rejected": -500.3805236816406,
+      "loss": 0.5826,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.4656105041503906,
+      "rewards/margins": 0.8493620157241821,
+      "rewards/rejected": -2.314972400665283,
+      "step": 2840
+    },
+    {
+      "epoch": 0.41954953628735464,
+      "grad_norm": 133.59046261362076,
+      "learning_rate": 3.599980608076312e-07,
+      "logits/chosen": -1.7649234533309937,
+      "logits/rejected": -1.5637601613998413,
+      "logps/chosen": -408.22589111328125,
+      "logps/rejected": -450.2254943847656,
+      "loss": 0.5604,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.1920119524002075,
+      "rewards/margins": 1.0949840545654297,
+      "rewards/rejected": -2.2869958877563477,
+      "step": 2850
+    },
+    {
+      "epoch": 0.4210216399234506,
+      "grad_norm": 104.90643999171841,
+      "learning_rate": 3.5884286098067124e-07,
+      "logits/chosen": -1.9010881185531616,
+      "logits/rejected": -1.6664737462997437,
+      "logps/chosen": -481.55255126953125,
+      "logps/rejected": -533.0050659179688,
+      "loss": 0.467,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.788642168045044,
+      "rewards/margins": 0.8375269174575806,
+      "rewards/rejected": -2.626168727874756,
+      "step": 2860
+    },
+    {
+      "epoch": 0.4224937435595466,
+      "grad_norm": 127.93392264394856,
+      "learning_rate": 3.5768478647080315e-07,
+      "logits/chosen": -1.5535461902618408,
+      "logits/rejected": -1.6039278507232666,
+      "logps/chosen": -457.58758544921875,
+      "logps/rejected": -516.9219970703125,
+      "loss": 0.536,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6912761926651,
+      "rewards/margins": 0.9885860681533813,
+      "rewards/rejected": -2.6798622608184814,
+      "step": 2870
+    },
+    {
+      "epoch": 0.42396584719564256,
+      "grad_norm": 76.66951905944022,
+      "learning_rate": 3.565238678642957e-07,
+      "logits/chosen": -1.4828526973724365,
+      "logits/rejected": -1.0871570110321045,
+      "logps/chosen": -514.9000244140625,
+      "logps/rejected": -565.3880615234375,
+      "loss": 0.479,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -2.057191848754883,
+      "rewards/margins": 1.4782410860061646,
+      "rewards/rejected": -3.535432815551758,
+      "step": 2880
+    },
+    {
+      "epoch": 0.4254379508317386,
+      "grad_norm": 74.1073437013685,
+      "learning_rate": 3.55360135822534e-07,
+      "logits/chosen": -1.6502548456192017,
+      "logits/rejected": -1.4351156949996948,
+      "logps/chosen": -477.233642578125,
+      "logps/rejected": -571.50732421875,
+      "loss": 0.4825,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9168500900268555,
+      "rewards/margins": 1.1503159999847412,
+      "rewards/rejected": -3.0671660900115967,
+      "step": 2890
+    },
+    {
+      "epoch": 0.42691005446783453,
+      "grad_norm": 126.75011331732391,
+      "learning_rate": 3.541936210812095e-07,
+      "logits/chosen": -1.5748753547668457,
+      "logits/rejected": -1.2683310508728027,
+      "logps/chosen": -476.5843200683594,
+      "logps/rejected": -569.3049926757812,
+      "loss": 0.3947,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -2.0876541137695312,
+      "rewards/margins": 1.4470094442367554,
+      "rewards/rejected": -3.534663438796997,
+      "step": 2900
+    },
+    {
+      "epoch": 0.42838215810393054,
+      "grad_norm": 76.34029595140632,
+      "learning_rate": 3.5302435444950894e-07,
+      "logits/chosen": -1.5367224216461182,
+      "logits/rejected": -1.1965324878692627,
+      "logps/chosen": -472.9261779785156,
+      "logps/rejected": -614.3412475585938,
+      "loss": 0.6052,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.8904402256011963,
+      "rewards/margins": 1.9703643321990967,
+      "rewards/rejected": -3.860804319381714,
+      "step": 2910
+    },
+    {
+      "epoch": 0.4298542617400265,
+      "grad_norm": 99.61793313483508,
+      "learning_rate": 3.518523668092994e-07,
+      "logits/chosen": -1.2857729196548462,
+      "logits/rejected": -1.0071913003921509,
+      "logps/chosen": -548.1761474609375,
+      "logps/rejected": -625.6806030273438,
+      "loss": 0.5609,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.728156566619873,
+      "rewards/margins": 1.225595235824585,
+      "rewards/rejected": -3.9537525177001953,
+      "step": 2920
+    },
+    {
+      "epoch": 0.43132636537612246,
+      "grad_norm": 88.6332661953972,
+      "learning_rate": 3.506776891143138e-07,
+      "logits/chosen": -1.552356481552124,
+      "logits/rejected": -1.6167726516723633,
+      "logps/chosen": -448.96484375,
+      "logps/rejected": -624.7279052734375,
+      "loss": 0.5411,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.9730370044708252,
+      "rewards/margins": 1.0921411514282227,
+      "rewards/rejected": -3.065178155899048,
+      "step": 2930
+    },
+    {
+      "epoch": 0.43279846901221847,
+      "grad_norm": 83.40511332062242,
+      "learning_rate": 3.495003523893329e-07,
+      "logits/chosen": -1.5380442142486572,
+      "logits/rejected": -1.5943825244903564,
+      "logps/chosen": -439.9825134277344,
+      "logps/rejected": -508.53851318359375,
+      "loss": 0.5666,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.0246939659118652,
+      "rewards/margins": 0.7639306783676147,
+      "rewards/rejected": -2.7886245250701904,
+      "step": 2940
+    },
+    {
+      "epoch": 0.4342705726483144,
+      "grad_norm": 55.51987780128476,
+      "learning_rate": 3.4832038772936574e-07,
+      "logits/chosen": -1.8873517513275146,
+      "logits/rejected": -1.7539236545562744,
+      "logps/chosen": -405.72906494140625,
+      "logps/rejected": -465.75433349609375,
+      "loss": 0.5226,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.5367895364761353,
+      "rewards/margins": 0.5773477554321289,
+      "rewards/rejected": -2.1141371726989746,
+      "step": 2950
+    },
+    {
+      "epoch": 0.43574267628441044,
+      "grad_norm": 93.65796111677007,
+      "learning_rate": 3.471378262988288e-07,
+      "logits/chosen": -1.706993818283081,
+      "logits/rejected": -1.5690408945083618,
+      "logps/chosen": -467.59619140625,
+      "logps/rejected": -549.7493896484375,
+      "loss": 0.4985,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.867831826210022,
+      "rewards/margins": 1.3700368404388428,
+      "rewards/rejected": -3.237868547439575,
+      "step": 2960
+    },
+    {
+      "epoch": 0.4372147799205064,
+      "grad_norm": 96.3640973140334,
+      "learning_rate": 3.459526993307225e-07,
+      "logits/chosen": -1.6291742324829102,
+      "logits/rejected": -1.3714163303375244,
+      "logps/chosen": -563.1551513671875,
+      "logps/rejected": -566.9429931640625,
+      "loss": 0.6511,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.3630969524383545,
+      "rewards/margins": 0.7266470789909363,
+      "rewards/rejected": -3.0897438526153564,
+      "step": 2970
+    },
+    {
+      "epoch": 0.4386868835566024,
+      "grad_norm": 90.42039043244793,
+      "learning_rate": 3.4476503812580677e-07,
+      "logits/chosen": -1.7459440231323242,
+      "logits/rejected": -1.4759334325790405,
+      "logps/chosen": -484.9942321777344,
+      "logps/rejected": -576.6343383789062,
+      "loss": 0.5027,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -1.9247719049453735,
+      "rewards/margins": 1.0805116891860962,
+      "rewards/rejected": -3.0052835941314697,
+      "step": 2980
+    },
+    {
+      "epoch": 0.44015898719269836,
+      "grad_norm": 76.09557851880464,
+      "learning_rate": 3.4357487405177367e-07,
+      "logits/chosen": -1.689424753189087,
+      "logits/rejected": -1.6925474405288696,
+      "logps/chosen": -386.05340576171875,
+      "logps/rejected": -513.0780029296875,
+      "loss": 0.4676,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.5728152990341187,
+      "rewards/margins": 1.129425048828125,
+      "rewards/rejected": -2.702240228652954,
+      "step": 2990
+    },
+    {
+      "epoch": 0.44163109082879437,
+      "grad_norm": 93.808001505817,
+      "learning_rate": 3.423822385424195e-07,
+      "logits/chosen": -1.7472833395004272,
+      "logits/rejected": -1.7409921884536743,
+      "logps/chosen": -445.745849609375,
+      "logps/rejected": -571.5545043945312,
+      "loss": 0.4092,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.7007602453231812,
+      "rewards/margins": 1.3439972400665283,
+      "rewards/rejected": -3.04475736618042,
+      "step": 3000
+    },
+    {
+      "epoch": 0.4431031944648903,
+      "grad_norm": 61.08186229678611,
+      "learning_rate": 3.411871630968145e-07,
+      "logits/chosen": -1.5771856307983398,
+      "logits/rejected": -1.3157498836517334,
+      "logps/chosen": -528.1099853515625,
+      "logps/rejected": -744.0299072265625,
+      "loss": 0.4846,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -2.2174181938171387,
+      "rewards/margins": 2.1569695472717285,
+      "rewards/rejected": -4.374387264251709,
+      "step": 3010
+    },
+    {
+      "epoch": 0.4445752981009863,
+      "grad_norm": 152.75349645859527,
+      "learning_rate": 3.3998967927847067e-07,
+      "logits/chosen": -1.6811447143554688,
+      "logits/rejected": -1.5367097854614258,
+      "logps/chosen": -531.6951904296875,
+      "logps/rejected": -636.5330200195312,
+      "loss": 0.5657,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -1.935083031654358,
+      "rewards/margins": 1.2397915124893188,
+      "rewards/rejected": -3.1748743057250977,
+      "step": 3020
+    },
+    {
+      "epoch": 0.4460474017370823,
+      "grad_norm": 85.31949008343528,
+      "learning_rate": 3.3878981871450846e-07,
+      "logits/chosen": -1.7278846502304077,
+      "logits/rejected": -1.5852535963058472,
+      "logps/chosen": -497.8582458496094,
+      "logps/rejected": -600.8698120117188,
+      "loss": 0.5145,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.7871097326278687,
+      "rewards/margins": 0.9624664187431335,
+      "rewards/rejected": -2.7495760917663574,
+      "step": 3030
+    },
+    {
+      "epoch": 0.44751950537317825,
+      "grad_norm": 124.42465815879098,
+      "learning_rate": 3.375876130948211e-07,
+      "logits/chosen": -1.554373025894165,
+      "logits/rejected": -1.2637821435928345,
+      "logps/chosen": -499.805908203125,
+      "logps/rejected": -618.772705078125,
+      "loss": 0.4519,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -2.3066258430480957,
+      "rewards/margins": 1.3394854068756104,
+      "rewards/rejected": -3.6461117267608643,
+      "step": 3040
+    },
+    {
+      "epoch": 0.44899160900927426,
+      "grad_norm": 123.63696424661978,
+      "learning_rate": 3.3638309417123824e-07,
+      "logits/chosen": -1.5253585577011108,
+      "logits/rejected": -1.2020318508148193,
+      "logps/chosen": -526.9351806640625,
+      "logps/rejected": -545.9700927734375,
+      "loss": 0.5424,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -2.3755273818969727,
+      "rewards/margins": 0.7981761693954468,
+      "rewards/rejected": -3.173703670501709,
+      "step": 3050
+    },
+    {
+      "epoch": 0.4504637126453702,
+      "grad_norm": 75.00017482503124,
+      "learning_rate": 3.3517629375668615e-07,
+      "logits/chosen": -1.6559574604034424,
+      "logits/rejected": -1.4047844409942627,
+      "logps/chosen": -521.6126098632812,
+      "logps/rejected": -628.2869873046875,
+      "loss": 0.4713,
+      "rewards/accuracies": 0.7333332300186157,
+      "rewards/chosen": -2.304652452468872,
+      "rewards/margins": 1.2449811697006226,
+      "rewards/rejected": -3.549633502960205,
+      "step": 3060
+    },
+    {
+      "epoch": 0.45193581628146623,
+      "grad_norm": 87.53956831154406,
+      "learning_rate": 3.3396724372434914e-07,
+      "logits/chosen": -1.7649757862091064,
+      "logits/rejected": -1.41666579246521,
+      "logps/chosen": -513.6778564453125,
+      "logps/rejected": -516.7106323242188,
+      "loss": 0.502,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.6286847591400146,
+      "rewards/margins": 1.3104242086410522,
+      "rewards/rejected": -2.9391090869903564,
+      "step": 3070
+    },
+    {
+      "epoch": 0.4534079199175622,
+      "grad_norm": 96.42195669627209,
+      "learning_rate": 3.327559760068263e-07,
+      "logits/chosen": -1.2842210531234741,
+      "logits/rejected": -1.3093006610870361,
+      "logps/chosen": -361.3973693847656,
+      "logps/rejected": -591.5134887695312,
+      "loss": 0.5441,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7170593738555908,
+      "rewards/margins": 1.7197749614715576,
+      "rewards/rejected": -3.4368338584899902,
+      "step": 3080
+    },
+    {
+      "epoch": 0.4548800235536582,
+      "grad_norm": 125.78559949794136,
+      "learning_rate": 3.3154252259528883e-07,
+      "logits/chosen": -1.7974132299423218,
+      "logits/rejected": -1.365867257118225,
+      "logps/chosen": -489.73651123046875,
+      "logps/rejected": -562.1240234375,
+      "loss": 0.4368,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.535992980003357,
+      "rewards/margins": 1.3945261240005493,
+      "rewards/rejected": -2.930518865585327,
+      "step": 3090
+    },
+    {
+      "epoch": 0.45635212718975415,
+      "grad_norm": 132.73671307201735,
+      "learning_rate": 3.303269155386351e-07,
+      "logits/chosen": -1.588967204093933,
+      "logits/rejected": -1.541374921798706,
+      "logps/chosen": -450.5350036621094,
+      "logps/rejected": -591.4512329101562,
+      "loss": 0.515,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.714499831199646,
+      "rewards/margins": 1.3883813619613647,
+      "rewards/rejected": -3.1028809547424316,
+      "step": 3100
+    },
+    {
+      "epoch": 0.45782423082585016,
+      "grad_norm": 84.35226676195592,
+      "learning_rate": 3.291091869426439e-07,
+      "logits/chosen": -1.4360454082489014,
+      "logits/rejected": -1.4084079265594482,
+      "logps/chosen": -489.7723693847656,
+      "logps/rejected": -656.9114990234375,
+      "loss": 0.4784,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -1.9550676345825195,
+      "rewards/margins": 2.1726884841918945,
+      "rewards/rejected": -4.127756118774414,
+      "step": 3110
+    },
+    {
+      "epoch": 0.4592963344619461,
+      "grad_norm": 138.7719561127843,
+      "learning_rate": 3.27889368969127e-07,
+      "logits/chosen": -0.8632322549819946,
+      "logits/rejected": -0.5557774305343628,
+      "logps/chosen": -555.629150390625,
+      "logps/rejected": -620.8408813476562,
+      "loss": 0.4959,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -2.7310373783111572,
+      "rewards/margins": 1.4438550472259521,
+      "rewards/rejected": -4.174892425537109,
+      "step": 3120
+    },
+    {
+      "epoch": 0.4607684380980421,
+      "grad_norm": 86.88693560595163,
+      "learning_rate": 3.266674938350789e-07,
+      "logits/chosen": -0.8334887623786926,
+      "logits/rejected": -0.2807777523994446,
+      "logps/chosen": -621.770751953125,
+      "logps/rejected": -709.0192260742188,
+      "loss": 0.4996,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -3.182051181793213,
+      "rewards/margins": 1.3985031843185425,
+      "rewards/rejected": -4.580554485321045,
+      "step": 3130
+    },
+    {
+      "epoch": 0.4622405417341381,
+      "grad_norm": 101.13490739996732,
+      "learning_rate": 3.254435938118269e-07,
+      "logits/chosen": -0.32349011301994324,
+      "logits/rejected": -1.0005395412445068,
+      "logps/chosen": -595.7144165039062,
+      "logps/rejected": -687.2969970703125,
+      "loss": 0.5751,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.945071220397949,
+      "rewards/margins": 0.8556967973709106,
+      "rewards/rejected": -3.8007683753967285,
+      "step": 3140
+    },
+    {
+      "epoch": 0.46371264537023404,
+      "grad_norm": 112.52931558596427,
+      "learning_rate": 3.242177012241778e-07,
+      "logits/chosen": -0.6440261602401733,
+      "logits/rejected": 0.2714475989341736,
+      "logps/chosen": -611.9390258789062,
+      "logps/rejected": -653.388916015625,
+      "loss": 0.5795,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -3.470188856124878,
+      "rewards/margins": 1.012743592262268,
+      "rewards/rejected": -4.482932090759277,
+      "step": 3150
+    },
+    {
+      "epoch": 0.46518474900633006,
+      "grad_norm": 98.97133750890707,
+      "learning_rate": 3.229898484495649e-07,
+      "logits/chosen": -0.5198215246200562,
+      "logits/rejected": -0.6536621451377869,
+      "logps/chosen": -457.37518310546875,
+      "logps/rejected": -566.5096435546875,
+      "loss": 0.5241,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -2.681630849838257,
+      "rewards/margins": 0.9847855567932129,
+      "rewards/rejected": -3.666416883468628,
+      "step": 3160
+    },
+    {
+      "epoch": 0.466656852642426,
+      "grad_norm": 74.33658755806334,
+      "learning_rate": 3.2176006791719266e-07,
+      "logits/chosen": -0.8453855514526367,
+      "logits/rejected": -0.7567756772041321,
+      "logps/chosen": -488.2428283691406,
+      "logps/rejected": -611.7319946289062,
+      "loss": 0.4892,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.6312668323516846,
+      "rewards/margins": 1.1969882249832153,
+      "rewards/rejected": -3.8282551765441895,
+      "step": 3170
+    },
+    {
+      "epoch": 0.468128956278522,
+      "grad_norm": 172.69730178456533,
+      "learning_rate": 3.2052839210718007e-07,
+      "logits/chosen": -0.9820365905761719,
+      "logits/rejected": -1.2467740774154663,
+      "logps/chosen": -452.60357666015625,
+      "logps/rejected": -584.418212890625,
+      "loss": 0.4949,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.2015039920806885,
+      "rewards/margins": 0.8768129348754883,
+      "rewards/rejected": -3.078317165374756,
+      "step": 3180
+    },
+    {
+      "epoch": 0.469601059914618,
+      "grad_norm": 63.62100677922358,
+      "learning_rate": 3.1929485354970297e-07,
+      "logits/chosen": -1.4609642028808594,
+      "logits/rejected": -1.1372168064117432,
+      "logps/chosen": -481.2571716308594,
+      "logps/rejected": -655.3583374023438,
+      "loss": 0.3664,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.9624278545379639,
+      "rewards/margins": 1.7153165340423584,
+      "rewards/rejected": -3.677743911743164,
+      "step": 3190
+    },
+    {
+      "epoch": 0.471073163550714,
+      "grad_norm": 60.70618218289404,
+      "learning_rate": 3.180594848241346e-07,
+      "logits/chosen": -1.1690118312835693,
+      "logits/rejected": -0.9623022079467773,
+      "logps/chosen": -400.1407470703125,
+      "logps/rejected": -662.8094482421875,
+      "loss": 0.557,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.7614247798919678,
+      "rewards/margins": 1.2712466716766357,
+      "rewards/rejected": -3.0326714515686035,
+      "step": 3200
+    },
+    {
+      "epoch": 0.47254526718680995,
+      "grad_norm": 61.67686186772589,
+      "learning_rate": 3.1682231855818565e-07,
+      "logits/chosen": -0.9251901507377625,
+      "logits/rejected": -0.8875066041946411,
+      "logps/chosen": -382.7342224121094,
+      "logps/rejected": -549.8390502929688,
+      "loss": 0.4504,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.8720834255218506,
+      "rewards/margins": 1.461350440979004,
+      "rewards/rejected": -3.3334336280822754,
+      "step": 3210
+    },
+    {
+      "epoch": 0.47401737082290596,
+      "grad_norm": 97.98450699752662,
+      "learning_rate": 3.1558338742704175e-07,
+      "logits/chosen": -1.0002073049545288,
+      "logits/rejected": -0.7228268384933472,
+      "logps/chosen": -517.6812133789062,
+      "logps/rejected": -538.5872802734375,
+      "loss": 0.6334,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.0547587871551514,
+      "rewards/margins": 0.9009310007095337,
+      "rewards/rejected": -2.9556899070739746,
+      "step": 3220
+    },
+    {
+      "epoch": 0.4754894744590019,
+      "grad_norm": 92.16897393499394,
+      "learning_rate": 3.1434272415250164e-07,
+      "logits/chosen": -1.1355243921279907,
+      "logits/rejected": -0.7260192632675171,
+      "logps/chosen": -496.72674560546875,
+      "logps/rejected": -601.2373046875,
+      "loss": 0.5128,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.9694559574127197,
+      "rewards/margins": 1.2598377466201782,
+      "rewards/rejected": -3.2292933464050293,
+      "step": 3230
+    },
+    {
+      "epoch": 0.47696157809509787,
+      "grad_norm": 46.80895832139151,
+      "learning_rate": 3.1310036150211155e-07,
+      "logits/chosen": -1.0410677194595337,
+      "logits/rejected": -0.8257973790168762,
+      "logps/chosen": -439.0824279785156,
+      "logps/rejected": -550.9937744140625,
+      "loss": 0.4721,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.5329301357269287,
+      "rewards/margins": 1.3073517084121704,
+      "rewards/rejected": -2.8402817249298096,
+      "step": 3240
+    },
+    {
+      "epoch": 0.4784336817311939,
+      "grad_norm": 87.95062826503703,
+      "learning_rate": 3.1185633228830076e-07,
+      "logits/chosen": -1.271874189376831,
+      "logits/rejected": -0.41823306679725647,
+      "logps/chosen": -457.9461364746094,
+      "logps/rejected": -551.9237060546875,
+      "loss": 0.4721,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.5670020580291748,
+      "rewards/margins": 1.43280827999115,
+      "rewards/rejected": -2.9998106956481934,
+      "step": 3250
+    },
+    {
+      "epoch": 0.47990578536728984,
+      "grad_norm": 66.92881169429357,
+      "learning_rate": 3.1061066936751483e-07,
+      "logits/chosen": -0.5986589193344116,
+      "logits/rejected": -0.36469608545303345,
+      "logps/chosen": -452.1084899902344,
+      "logps/rejected": -606.2619018554688,
+      "loss": 0.4009,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -1.9342241287231445,
+      "rewards/margins": 1.2434868812561035,
+      "rewards/rejected": -3.177711009979248,
+      "step": 3260
+    },
+    {
+      "epoch": 0.48137788900338585,
+      "grad_norm": 103.75134336547266,
+      "learning_rate": 3.0936340563934733e-07,
+      "logits/chosen": -0.35131606459617615,
+      "logits/rejected": 0.34717637300491333,
+      "logps/chosen": -446.91717529296875,
+      "logps/rejected": -612.5995483398438,
+      "loss": 0.5381,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.9126863479614258,
+      "rewards/margins": 1.7402470111846924,
+      "rewards/rejected": -3.6529335975646973,
+      "step": 3270
+    },
+    {
+      "epoch": 0.4828499926394818,
+      "grad_norm": 77.8948154573654,
+      "learning_rate": 3.0811457404567153e-07,
+      "logits/chosen": -0.7472583055496216,
+      "logits/rejected": 0.11789913475513458,
+      "logps/chosen": -519.041015625,
+      "logps/rejected": -527.4039916992188,
+      "loss": 0.4594,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.0146710872650146,
+      "rewards/margins": 0.8171228170394897,
+      "rewards/rejected": -2.831794023513794,
+      "step": 3280
+    },
+    {
+      "epoch": 0.4843220962755778,
+      "grad_norm": 167.9212084252121,
+      "learning_rate": 3.068642075697699e-07,
+      "logits/chosen": -0.42541250586509705,
+      "logits/rejected": 0.21263058483600616,
+      "logps/chosen": -503.61920166015625,
+      "logps/rejected": -599.8175048828125,
+      "loss": 0.6,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -2.501636505126953,
+      "rewards/margins": 1.1055630445480347,
+      "rewards/rejected": -3.6071999073028564,
+      "step": 3290
+    },
+    {
+      "epoch": 0.4857941999116738,
+      "grad_norm": 168.07406222164585,
+      "learning_rate": 3.0561233923546336e-07,
+      "logits/chosen": -0.4212028980255127,
+      "logits/rejected": 0.12252716720104218,
+      "logps/chosen": -459.73614501953125,
+      "logps/rejected": -608.9522705078125,
+      "loss": 0.51,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.2331066131591797,
+      "rewards/margins": 1.4988734722137451,
+      "rewards/rejected": -3.731980085372925,
+      "step": 3300
+    },
+    {
+      "epoch": 0.4872663035477698,
+      "grad_norm": 72.64696076498669,
+      "learning_rate": 3.0435900210623867e-07,
+      "logits/chosen": -0.9463162422180176,
+      "logits/rejected": -0.13293954730033875,
+      "logps/chosen": -473.06561279296875,
+      "logps/rejected": -623.835205078125,
+      "loss": 0.4109,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -2.18930983543396,
+      "rewards/margins": 1.4071576595306396,
+      "rewards/rejected": -3.596467971801758,
+      "step": 3310
+    },
+    {
+      "epoch": 0.48873840718386574,
+      "grad_norm": 76.13872340767301,
+      "learning_rate": 3.031042292843755e-07,
+      "logits/chosen": -0.7620629668235779,
+      "logits/rejected": -0.11279682070016861,
+      "logps/chosen": -505.63043212890625,
+      "logps/rejected": -617.1547241210938,
+      "loss": 0.5179,
+      "rewards/accuracies": 0.9000000953674316,
+      "rewards/chosen": -2.258166790008545,
+      "rewards/margins": 1.1824322938919067,
+      "rewards/rejected": -3.440598964691162,
+      "step": 3320
+    },
+    {
+      "epoch": 0.4902105108199617,
+      "grad_norm": 124.287656631569,
+      "learning_rate": 3.0184805391007205e-07,
+      "logits/chosen": -0.7151976227760315,
+      "logits/rejected": -0.8751751184463501,
+      "logps/chosen": -550.53759765625,
+      "logps/rejected": -651.2049560546875,
+      "loss": 0.441,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -2.647606611251831,
+      "rewards/margins": 0.7530060410499573,
+      "rewards/rejected": -3.4006130695343018,
+      "step": 3330
+    },
+    {
+      "epoch": 0.4916826144560577,
+      "grad_norm": 135.07782979997054,
+      "learning_rate": 3.0059050916056977e-07,
+      "logits/chosen": -1.12015700340271,
+      "logits/rejected": -0.4860397279262543,
+      "logps/chosen": -518.6348876953125,
+      "logps/rejected": -528.7443237304688,
+      "loss": 0.5541,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.6023058891296387,
+      "rewards/margins": 0.8805994987487793,
+      "rewards/rejected": -3.4829049110412598,
+      "step": 3340
+    },
+    {
+      "epoch": 0.49315471809215367,
+      "grad_norm": 227.43220372603295,
+      "learning_rate": 2.9933162824927713e-07,
+      "logits/chosen": -1.0092880725860596,
+      "logits/rejected": -0.72817462682724,
+      "logps/chosen": -446.1585998535156,
+      "logps/rejected": -555.7626953125,
+      "loss": 0.581,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -2.2049596309661865,
+      "rewards/margins": 1.1553776264190674,
+      "rewards/rejected": -3.360337495803833,
+      "step": 3350
+    },
+    {
+      "epoch": 0.4946268217282497,
+      "grad_norm": 113.59146684583767,
+      "learning_rate": 2.9807144442489234e-07,
+      "logits/chosen": -1.2836788892745972,
+      "logits/rejected": -1.0262157917022705,
+      "logps/chosen": -513.675537109375,
+      "logps/rejected": -585.2378540039062,
+      "loss": 0.4447,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -2.151932716369629,
+      "rewards/margins": 0.9616401791572571,
+      "rewards/rejected": -3.113572835922241,
+      "step": 3360
+    },
+    {
+      "epoch": 0.49609892536434563,
+      "grad_norm": 151.2761705515275,
+      "learning_rate": 2.9680999097052536e-07,
+      "logits/chosen": -0.9294954538345337,
+      "logits/rejected": -0.5569708943367004,
+      "logps/chosen": -420.66619873046875,
+      "logps/rejected": -587.71435546875,
+      "loss": 0.479,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -2.234401226043701,
+      "rewards/margins": 1.5371743440628052,
+      "rewards/rejected": -3.771575450897217,
+      "step": 3370
+    },
+    {
+      "epoch": 0.49757102900044164,
+      "grad_norm": 163.77758518356075,
+      "learning_rate": 2.9554730120281847e-07,
+      "logits/chosen": -0.9432289004325867,
+      "logits/rejected": -0.5820508599281311,
+      "logps/chosen": -520.8422241210938,
+      "logps/rejected": -579.085693359375,
+      "loss": 0.4814,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.5879578590393066,
+      "rewards/margins": 1.0231280326843262,
+      "rewards/rejected": -3.611085891723633,
+      "step": 3380
+    },
+    {
+      "epoch": 0.4990431326365376,
+      "grad_norm": 118.552399882384,
+      "learning_rate": 2.942834084710668e-07,
+      "logits/chosen": -0.892548680305481,
+      "logits/rejected": -0.5368214845657349,
+      "logps/chosen": -583.15478515625,
+      "logps/rejected": -600.8814697265625,
+      "loss": 0.5559,
+      "rewards/accuracies": 0.5666666626930237,
+      "rewards/chosen": -3.060138702392578,
+      "rewards/margins": 0.7581546306610107,
+      "rewards/rejected": -3.8182930946350098,
+      "step": 3390
+    },
+    {
+      "epoch": 0.5005152362726336,
+      "grad_norm": 61.18321683249202,
+      "learning_rate": 2.9301834615633766e-07,
+      "logits/chosen": -0.7124353051185608,
+      "logits/rejected": -0.7218102216720581,
+      "logps/chosen": -447.5498046875,
+      "logps/rejected": -578.6387939453125,
+      "loss": 0.5324,
+      "rewards/accuracies": 0.7999998927116394,
+      "rewards/chosen": -2.3925974369049072,
+      "rewards/margins": 1.289864182472229,
+      "rewards/rejected": -3.682461977005005,
+      "step": 3400
+    },
+    {
+      "epoch": 0.5019873399087296,
+      "grad_norm": 70.14826603763085,
+      "learning_rate": 2.917521476705879e-07,
+      "logits/chosen": -1.1851778030395508,
+      "logits/rejected": -0.8059045672416687,
+      "logps/chosen": -569.7425537109375,
+      "logps/rejected": -691.015869140625,
+      "loss": 0.4258,
+      "rewards/accuracies": 0.9000000953674316,
+      "rewards/chosen": -2.613755702972412,
+      "rewards/margins": 1.6566842794418335,
+      "rewards/rejected": -4.270440101623535,
+      "step": 3410
+    },
+    {
+      "epoch": 0.5034594435448255,
+      "grad_norm": 45.652088146296705,
+      "learning_rate": 2.904848464557827e-07,
+      "logits/chosen": -0.9869047403335571,
+      "logits/rejected": -0.5741862654685974,
+      "logps/chosen": -577.0040283203125,
+      "logps/rejected": -618.1246948242188,
+      "loss": 0.5622,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -2.937192916870117,
+      "rewards/margins": 0.7588188052177429,
+      "rewards/rejected": -3.6960113048553467,
+      "step": 3420
+    },
+    {
+      "epoch": 0.5049315471809216,
+      "grad_norm": 93.7953203447838,
+      "learning_rate": 2.892164759830114e-07,
+      "logits/chosen": -0.8094015121459961,
+      "logits/rejected": -0.47796908020973206,
+      "logps/chosen": -516.6316528320312,
+      "logps/rejected": -595.246826171875,
+      "loss": 0.4979,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.5407001972198486,
+      "rewards/margins": 0.8587757349014282,
+      "rewards/rejected": -3.3994762897491455,
+      "step": 3430
+    },
+    {
+      "epoch": 0.5064036508170175,
+      "grad_norm": 86.41127829758365,
+      "learning_rate": 2.8794706975160425e-07,
+      "logits/chosen": -0.7042872309684753,
+      "logits/rejected": 0.20396976172924042,
+      "logps/chosen": -495.3429260253906,
+      "logps/rejected": -618.5006713867188,
+      "loss": 0.5474,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.5355000495910645,
+      "rewards/margins": 1.2149041891098022,
+      "rewards/rejected": -3.750403642654419,
+      "step": 3440
+    },
+    {
+      "epoch": 0.5078757544531135,
+      "grad_norm": 164.93603292789112,
+      "learning_rate": 2.866766612882468e-07,
+      "logits/chosen": -1.3000671863555908,
+      "logits/rejected": -0.9017229080200195,
+      "logps/chosen": -479.858154296875,
+      "logps/rejected": -591.9417724609375,
+      "loss": 0.4891,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.1474249362945557,
+      "rewards/margins": 0.9466966390609741,
+      "rewards/rejected": -3.094121217727661,
+      "step": 3450
+    },
+    {
+      "epoch": 0.5093478580892095,
+      "grad_norm": 96.37561146231899,
+      "learning_rate": 2.8540528414609514e-07,
+      "logits/chosen": -1.018848180770874,
+      "logits/rejected": -0.7929521799087524,
+      "logps/chosen": -447.865234375,
+      "logps/rejected": -651.18701171875,
+      "loss": 0.4804,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.2059645652770996,
+      "rewards/margins": 1.5832328796386719,
+      "rewards/rejected": -3.7891972064971924,
+      "step": 3460
+    },
+    {
+      "epoch": 0.5108199617253054,
+      "grad_norm": 196.46591386606642,
+      "learning_rate": 2.8413297190388947e-07,
+      "logits/chosen": -1.0635197162628174,
+      "logits/rejected": -0.8645036816596985,
+      "logps/chosen": -504.1944885253906,
+      "logps/rejected": -570.5606689453125,
+      "loss": 0.4855,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -2.526508092880249,
+      "rewards/margins": 0.9527558088302612,
+      "rewards/rejected": -3.4792640209198,
+      "step": 3470
+    },
+    {
+      "epoch": 0.5122920653614015,
+      "grad_norm": 133.05840098255638,
+      "learning_rate": 2.8285975816506687e-07,
+      "logits/chosen": -1.500643253326416,
+      "logits/rejected": -1.4915549755096436,
+      "logps/chosen": -579.1746215820312,
+      "logps/rejected": -564.9222412109375,
+      "loss": 0.4829,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.0425267219543457,
+      "rewards/margins": 0.6850039958953857,
+      "rewards/rejected": -2.7275304794311523,
+      "step": 3480
+    },
+    {
+      "epoch": 0.5137641689974974,
+      "grad_norm": 114.26230847706957,
+      "learning_rate": 2.815856765568746e-07,
+      "logits/chosen": -1.5373557806015015,
+      "logits/rejected": -1.3047198057174683,
+      "logps/chosen": -454.78509521484375,
+      "logps/rejected": -468.45477294921875,
+      "loss": 0.4396,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6142804622650146,
+      "rewards/margins": 0.8521246910095215,
+      "rewards/rejected": -2.466405153274536,
+      "step": 3490
+    },
+    {
+      "epoch": 0.5152362726335934,
+      "grad_norm": 149.07762686642013,
+      "learning_rate": 2.803107607294811e-07,
+      "logits/chosen": -1.3443024158477783,
+      "logits/rejected": -0.9240787625312805,
+      "logps/chosen": -468.52276611328125,
+      "logps/rejected": -563.1370239257812,
+      "loss": 0.6109,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.1036009788513184,
+      "rewards/margins": 1.0239379405975342,
+      "rewards/rejected": -3.1275384426116943,
+      "step": 3500
+    },
+    {
+      "epoch": 0.5167083762696894,
+      "grad_norm": 60.07938664565102,
+      "learning_rate": 2.7903504435508787e-07,
+      "logits/chosen": -1.2649275064468384,
+      "logits/rejected": -1.0444282293319702,
+      "logps/chosen": -524.1019287109375,
+      "logps/rejected": -608.5594482421875,
+      "loss": 0.6044,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1403751373291016,
+      "rewards/margins": 1.0087820291519165,
+      "rewards/rejected": -3.1491570472717285,
+      "step": 3510
+    },
+    {
+      "epoch": 0.5181804799057854,
+      "grad_norm": 82.85979150873672,
+      "learning_rate": 2.777585611270398e-07,
+      "logits/chosen": -1.093212604522705,
+      "logits/rejected": -0.8458244204521179,
+      "logps/chosen": -476.87091064453125,
+      "logps/rejected": -545.1942138671875,
+      "loss": 0.5018,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.1044373512268066,
+      "rewards/margins": 1.140057921409607,
+      "rewards/rejected": -3.244494676589966,
+      "step": 3520
+    },
+    {
+      "epoch": 0.5196525835418814,
+      "grad_norm": 132.8237267241224,
+      "learning_rate": 2.7648134475893544e-07,
+      "logits/chosen": -1.1214923858642578,
+      "logits/rejected": -0.8191500902175903,
+      "logps/chosen": -443.44964599609375,
+      "logps/rejected": -497.73419189453125,
+      "loss": 0.5932,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0088188648223877,
+      "rewards/margins": 0.6817129850387573,
+      "rewards/rejected": -2.6905322074890137,
+      "step": 3530
+    },
+    {
+      "epoch": 0.5211246871779773,
+      "grad_norm": 101.9469424248708,
+      "learning_rate": 2.7520342898373657e-07,
+      "logits/chosen": -0.797698974609375,
+      "logits/rejected": -0.7146264314651489,
+      "logps/chosen": -411.5892639160156,
+      "logps/rejected": -486.6763610839844,
+      "loss": 0.5879,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -2.038975954055786,
+      "rewards/margins": 0.7395116090774536,
+      "rewards/rejected": -2.7784876823425293,
+      "step": 3540
+    },
+    {
+      "epoch": 0.5225967908140733,
+      "grad_norm": 135.8145510125049,
+      "learning_rate": 2.73924847552877e-07,
+      "logits/chosen": -1.0406383275985718,
+      "logits/rejected": -0.761631965637207,
+      "logps/chosen": -420.68206787109375,
+      "logps/rejected": -464.0370178222656,
+      "loss": 0.5155,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6280235052108765,
+      "rewards/margins": 0.8213141560554504,
+      "rewards/rejected": -2.449337959289551,
+      "step": 3550
+    },
+    {
+      "epoch": 0.5240688944501692,
+      "grad_norm": 40.68831168502621,
+      "learning_rate": 2.726456342353718e-07,
+      "logits/chosen": -1.4041072130203247,
+      "logits/rejected": -1.2757805585861206,
+      "logps/chosen": -485.3564453125,
+      "logps/rejected": -557.1806030273438,
+      "loss": 0.4311,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.7481788396835327,
+      "rewards/margins": 0.8918279409408569,
+      "rewards/rejected": -2.6400065422058105,
+      "step": 3560
+    },
+    {
+      "epoch": 0.5255409980862653,
+      "grad_norm": 86.82123480554844,
+      "learning_rate": 2.7136582281692443e-07,
+      "logits/chosen": -0.9888653755187988,
+      "logits/rejected": -0.7883027195930481,
+      "logps/chosen": -455.83990478515625,
+      "logps/rejected": -528.5462646484375,
+      "loss": 0.4741,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.8590036630630493,
+      "rewards/margins": 1.416704773902893,
+      "rewards/rejected": -3.2757084369659424,
+      "step": 3570
+    },
+    {
+      "epoch": 0.5270131017223613,
+      "grad_norm": 66.2519746442281,
+      "learning_rate": 2.700854470990357e-07,
+      "logits/chosen": -0.8329097032546997,
+      "logits/rejected": -0.5545793175697327,
+      "logps/chosen": -493.0042419433594,
+      "logps/rejected": -610.2681274414062,
+      "loss": 0.548,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -2.285214900970459,
+      "rewards/margins": 1.2870739698410034,
+      "rewards/rejected": -3.572288990020752,
+      "step": 3580
+    },
+    {
+      "epoch": 0.5284852053584572,
+      "grad_norm": 94.05341844759623,
+      "learning_rate": 2.6880454089810954e-07,
+      "logits/chosen": -1.2370531558990479,
+      "logits/rejected": -0.6671432256698608,
+      "logps/chosen": -491.1453552246094,
+      "logps/rejected": -555.08203125,
+      "loss": 0.5699,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.9788224697113037,
+      "rewards/margins": 0.8359730839729309,
+      "rewards/rejected": -2.8147952556610107,
+      "step": 3590
+    },
+    {
+      "epoch": 0.5299573089945532,
+      "grad_norm": 106.15463681434012,
+      "learning_rate": 2.6752313804456124e-07,
+      "logits/chosen": -1.1055041551589966,
+      "logits/rejected": -0.6545578241348267,
+      "logps/chosen": -398.18572998046875,
+      "logps/rejected": -457.91021728515625,
+      "loss": 0.4474,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -1.7753149271011353,
+      "rewards/margins": 1.1495778560638428,
+      "rewards/rejected": -2.9248929023742676,
+      "step": 3600
+    },
+    {
+      "epoch": 0.5314294126306492,
+      "grad_norm": 51.42329147526349,
+      "learning_rate": 2.66241272381923e-07,
+      "logits/chosen": -1.4011653661727905,
+      "logits/rejected": -0.9820989370346069,
+      "logps/chosen": -487.7442321777344,
+      "logps/rejected": -488.4327697753906,
+      "loss": 0.475,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.4035143852233887,
+      "rewards/margins": 0.5535462498664856,
+      "rewards/rejected": -2.9570605754852295,
+      "step": 3610
+    },
+    {
+      "epoch": 0.5329015162667452,
+      "grad_norm": 83.07618658077477,
+      "learning_rate": 2.649589777659506e-07,
+      "logits/chosen": -1.0022094249725342,
+      "logits/rejected": -0.9162474870681763,
+      "logps/chosen": -403.79388427734375,
+      "logps/rejected": -632.6209716796875,
+      "loss": 0.4923,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -2.006913423538208,
+      "rewards/margins": 1.435204267501831,
+      "rewards/rejected": -3.442117691040039,
+      "step": 3620
+    },
+    {
+      "epoch": 0.5343736199028412,
+      "grad_norm": 95.48613411816375,
+      "learning_rate": 2.6367628806372893e-07,
+      "logits/chosen": -1.3434334993362427,
+      "logits/rejected": -0.6106802821159363,
+      "logps/chosen": -589.4462280273438,
+      "logps/rejected": -620.58349609375,
+      "loss": 0.5333,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.4060192108154297,
+      "rewards/margins": 1.289965271949768,
+      "rewards/rejected": -3.695984363555908,
+      "step": 3630
+    },
+    {
+      "epoch": 0.5358457235389371,
+      "grad_norm": 88.94966670076559,
+      "learning_rate": 2.623932371527776e-07,
+      "logits/chosen": -1.4550727605819702,
+      "logits/rejected": -0.5974916219711304,
+      "logps/chosen": -512.2884521484375,
+      "logps/rejected": -567.3715209960938,
+      "loss": 0.5042,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9926986694335938,
+      "rewards/margins": 1.441896915435791,
+      "rewards/rejected": -3.434595823287964,
+      "step": 3640
+    },
+    {
+      "epoch": 0.5373178271750331,
+      "grad_norm": 144.70516586885373,
+      "learning_rate": 2.611098589201563e-07,
+      "logits/chosen": -0.8839343786239624,
+      "logits/rejected": -0.6502280831336975,
+      "logps/chosen": -454.4888610839844,
+      "logps/rejected": -609.1740112304688,
+      "loss": 0.4138,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -2.3977155685424805,
+      "rewards/margins": 1.6403570175170898,
+      "rewards/rejected": -4.0380730628967285,
+      "step": 3650
+    },
+    {
+      "epoch": 0.5387899308111291,
+      "grad_norm": 322.08883595138076,
+      "learning_rate": 2.5982618726156965e-07,
+      "logits/chosen": -1.1795523166656494,
+      "logits/rejected": -0.8610784411430359,
+      "logps/chosen": -436.20977783203125,
+      "logps/rejected": -571.6282958984375,
+      "loss": 0.4293,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -2.2965991497039795,
+      "rewards/margins": 1.3638889789581299,
+      "rewards/rejected": -3.6604881286621094,
+      "step": 3660
+    },
+    {
+      "epoch": 0.5402620344472251,
+      "grad_norm": 79.91620253175974,
+      "learning_rate": 2.5854225608047217e-07,
+      "logits/chosen": -1.0248043537139893,
+      "logits/rejected": -0.8329952955245972,
+      "logps/chosen": -502.764892578125,
+      "logps/rejected": -602.9068603515625,
+      "loss": 0.4291,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.441542625427246,
+      "rewards/margins": 1.0967991352081299,
+      "rewards/rejected": -3.538341522216797,
+      "step": 3670
+    },
+    {
+      "epoch": 0.541734138083321,
+      "grad_norm": 141.46441662539445,
+      "learning_rate": 2.572580992871725e-07,
+      "logits/chosen": -1.5260649919509888,
+      "logits/rejected": -1.1740323305130005,
+      "logps/chosen": -588.5630493164062,
+      "logps/rejected": -625.4890747070312,
+      "loss": 0.5023,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.2363507747650146,
+      "rewards/margins": 0.9393233060836792,
+      "rewards/rejected": -3.1756739616394043,
+      "step": 3680
+    },
+    {
+      "epoch": 0.543206241719417,
+      "grad_norm": 104.28666697073605,
+      "learning_rate": 2.5597375079793826e-07,
+      "logits/chosen": -1.2928651571273804,
+      "logits/rejected": -0.8324087262153625,
+      "logps/chosen": -412.5162048339844,
+      "logps/rejected": -518.4513549804688,
+      "loss": 0.5089,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.765946626663208,
+      "rewards/margins": 0.946916401386261,
+      "rewards/rejected": -2.712862730026245,
+      "step": 3690
+    },
+    {
+      "epoch": 0.5446783453555131,
+      "grad_norm": 130.83779984210676,
+      "learning_rate": 2.5468924453409977e-07,
+      "logits/chosen": -1.262528896331787,
+      "logits/rejected": -0.9617989659309387,
+      "logps/chosen": -470.62945556640625,
+      "logps/rejected": -536.2945556640625,
+      "loss": 0.7204,
+      "rewards/accuracies": 0.5333333015441895,
+      "rewards/chosen": -2.2758002281188965,
+      "rewards/margins": 0.6281536817550659,
+      "rewards/rejected": -2.903953790664673,
+      "step": 3700
+    },
+    {
+      "epoch": 0.546150448991609,
+      "grad_norm": 152.95755861823764,
+      "learning_rate": 2.534046144211544e-07,
+      "logits/chosen": -1.5449644327163696,
+      "logits/rejected": -1.4245322942733765,
+      "logps/chosen": -471.7792053222656,
+      "logps/rejected": -541.1217651367188,
+      "loss": 0.5859,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.8622760772705078,
+      "rewards/margins": 0.9120270609855652,
+      "rewards/rejected": -2.7743031978607178,
+      "step": 3710
+    },
+    {
+      "epoch": 0.547622552627705,
+      "grad_norm": 59.17155786640775,
+      "learning_rate": 2.521198943878708e-07,
+      "logits/chosen": -1.4385583400726318,
+      "logits/rejected": -1.7578184604644775,
+      "logps/chosen": -440.26641845703125,
+      "logps/rejected": -546.3853759765625,
+      "loss": 0.5733,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -1.8807086944580078,
+      "rewards/margins": 0.3880549967288971,
+      "rewards/rejected": -2.268763780593872,
+      "step": 3720
+    },
+    {
+      "epoch": 0.5490946562638009,
+      "grad_norm": 61.881472655644686,
+      "learning_rate": 2.5083511836539213e-07,
+      "logits/chosen": -1.77863347530365,
+      "logits/rejected": -1.6871923208236694,
+      "logps/chosen": -429.9437561035156,
+      "logps/rejected": -447.72186279296875,
+      "loss": 0.4463,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.5609115362167358,
+      "rewards/margins": 0.4274429380893707,
+      "rewards/rejected": -1.9883544445037842,
+      "step": 3730
+    },
+    {
+      "epoch": 0.550566759899897,
+      "grad_norm": 90.66992302484171,
+      "learning_rate": 2.495503202863407e-07,
+      "logits/chosen": -1.6335229873657227,
+      "logits/rejected": -1.4598299264907837,
+      "logps/chosen": -396.29583740234375,
+      "logps/rejected": -454.7091369628906,
+      "loss": 0.5254,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.6245256662368774,
+      "rewards/margins": 0.949206531047821,
+      "rewards/rejected": -2.5737321376800537,
+      "step": 3740
+    },
+    {
+      "epoch": 0.552038863535993,
+      "grad_norm": 45.57917448479132,
+      "learning_rate": 2.4826553408392104e-07,
+      "logits/chosen": -1.6143033504486084,
+      "logits/rejected": -1.2706949710845947,
+      "logps/chosen": -380.66241455078125,
+      "logps/rejected": -499.0823669433594,
+      "loss": 0.4934,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.3969308137893677,
+      "rewards/margins": 1.380838394165039,
+      "rewards/rejected": -2.7777693271636963,
+      "step": 3750
+    },
+    {
+      "epoch": 0.5535109671720889,
+      "grad_norm": 106.56612027161793,
+      "learning_rate": 2.469807936910242e-07,
+      "logits/chosen": -1.5957189798355103,
+      "logits/rejected": -1.3597112894058228,
+      "logps/chosen": -441.5704040527344,
+      "logps/rejected": -518.8150634765625,
+      "loss": 0.4738,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.4410152435302734,
+      "rewards/margins": 1.0514957904815674,
+      "rewards/rejected": -2.49251127243042,
+      "step": 3760
+    },
+    {
+      "epoch": 0.5549830708081849,
+      "grad_norm": 82.56240515825458,
+      "learning_rate": 2.456961330393313e-07,
+      "logits/chosen": -1.5211899280548096,
+      "logits/rejected": -1.0843480825424194,
+      "logps/chosen": -438.5062561035156,
+      "logps/rejected": -453.1576232910156,
+      "loss": 0.5257,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -1.4770877361297607,
+      "rewards/margins": 0.8786066174507141,
+      "rewards/rejected": -2.35569429397583,
+      "step": 3770
+    },
+    {
+      "epoch": 0.5564551744442808,
+      "grad_norm": 65.3512682214412,
+      "learning_rate": 2.444115860584174e-07,
+      "logits/chosen": -1.2876940965652466,
+      "logits/rejected": -0.9141214489936829,
+      "logps/chosen": -478.1373596191406,
+      "logps/rejected": -567.1768798828125,
+      "loss": 0.4454,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.8468248844146729,
+      "rewards/margins": 1.1089718341827393,
+      "rewards/rejected": -2.955796480178833,
+      "step": 3780
+    },
+    {
+      "epoch": 0.5579272780803769,
+      "grad_norm": 99.23231842653799,
+      "learning_rate": 2.4312718667485523e-07,
+      "logits/chosen": -1.4020931720733643,
+      "logits/rejected": -1.1753932237625122,
+      "logps/chosen": -404.2585144042969,
+      "logps/rejected": -488.93670654296875,
+      "loss": 0.4067,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -1.532779574394226,
+      "rewards/margins": 1.1545389890670776,
+      "rewards/rejected": -2.6873185634613037,
+      "step": 3790
+    },
+    {
+      "epoch": 0.5593993817164729,
+      "grad_norm": 100.61534189513759,
+      "learning_rate": 2.418429688113194e-07,
+      "logits/chosen": -1.2818669080734253,
+      "logits/rejected": -0.9343087077140808,
+      "logps/chosen": -598.4937744140625,
+      "logps/rejected": -568.2569580078125,
+      "loss": 0.4878,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.617891788482666,
+      "rewards/margins": 0.8785541653633118,
+      "rewards/rejected": -3.496445894241333,
+      "step": 3800
+    },
+    {
+      "epoch": 0.5608714853525688,
+      "grad_norm": 125.06370873034457,
+      "learning_rate": 2.405589663856904e-07,
+      "logits/chosen": -1.0534967184066772,
+      "logits/rejected": -0.796125054359436,
+      "logps/chosen": -456.1944274902344,
+      "logps/rejected": -579.9053955078125,
+      "loss": 0.462,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.9744675159454346,
+      "rewards/margins": 1.4265060424804688,
+      "rewards/rejected": -3.4009735584259033,
+      "step": 3810
+    },
+    {
+      "epoch": 0.5623435889886648,
+      "grad_norm": 59.73602878608555,
+      "learning_rate": 2.3927521331015865e-07,
+      "logits/chosen": -1.0593222379684448,
+      "logits/rejected": -0.7076674699783325,
+      "logps/chosen": -380.1196594238281,
+      "logps/rejected": -539.4362182617188,
+      "loss": 0.4875,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.722365140914917,
+      "rewards/margins": 1.5474141836166382,
+      "rewards/rejected": -3.2697787284851074,
+      "step": 3820
+    },
+    {
+      "epoch": 0.5638156926247608,
+      "grad_norm": 79.65260420030506,
+      "learning_rate": 2.379917434903289e-07,
+      "logits/chosen": -0.9202688336372375,
+      "logits/rejected": -0.8789048194885254,
+      "logps/chosen": -527.4368286132812,
+      "logps/rejected": -653.8619384765625,
+      "loss": 0.4841,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.5801780223846436,
+      "rewards/margins": 1.1735316514968872,
+      "rewards/rejected": -3.753709316253662,
+      "step": 3830
+    },
+    {
+      "epoch": 0.5652877962608568,
+      "grad_norm": 105.94838214690745,
+      "learning_rate": 2.3670859082432458e-07,
+      "logits/chosen": -1.366370439529419,
+      "logits/rejected": -1.1931499242782593,
+      "logps/chosen": -533.2824096679688,
+      "logps/rejected": -643.5277099609375,
+      "loss": 0.523,
+      "rewards/accuracies": 0.9000000953674316,
+      "rewards/chosen": -2.2342772483825684,
+      "rewards/margins": 1.7616726160049438,
+      "rewards/rejected": -3.995950222015381,
+      "step": 3840
+    },
+    {
+      "epoch": 0.5667598998969527,
+      "grad_norm": 66.40181477677648,
+      "learning_rate": 2.35425789201893e-07,
+      "logits/chosen": -0.9468305706977844,
+      "logits/rejected": -0.4881543219089508,
+      "logps/chosen": -535.8870239257812,
+      "logps/rejected": -688.7952270507812,
+      "loss": 0.416,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.5945944786071777,
+      "rewards/margins": 1.6216539144515991,
+      "rewards/rejected": -4.216248512268066,
+      "step": 3850
+    },
+    {
+      "epoch": 0.5682320035330487,
+      "grad_norm": 57.94401991168832,
+      "learning_rate": 2.3414337250350982e-07,
+      "logits/chosen": -0.9698936343193054,
+      "logits/rejected": -0.698264479637146,
+      "logps/chosen": -499.48114013671875,
+      "logps/rejected": -620.9559326171875,
+      "loss": 0.5635,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -2.667584180831909,
+      "rewards/margins": 1.292751669883728,
+      "rewards/rejected": -3.960336208343506,
+      "step": 3860
+    },
+    {
+      "epoch": 0.5697041071691447,
+      "grad_norm": 120.33478274769186,
+      "learning_rate": 2.3286137459948428e-07,
+      "logits/chosen": -1.1415274143218994,
+      "logits/rejected": -0.9294644594192505,
+      "logps/chosen": -384.8303527832031,
+      "logps/rejected": -570.9527587890625,
+      "loss": 0.5407,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.9156959056854248,
+      "rewards/margins": 1.6666879653930664,
+      "rewards/rejected": -3.582383632659912,
+      "step": 3870
+    },
+    {
+      "epoch": 0.5711762108052407,
+      "grad_norm": 113.35472760156007,
+      "learning_rate": 2.3157982934906463e-07,
+      "logits/chosen": -1.6657747030258179,
+      "logits/rejected": -1.5370306968688965,
+      "logps/chosen": -462.897705078125,
+      "logps/rejected": -558.5958862304688,
+      "loss": 0.4491,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.4978281259536743,
+      "rewards/margins": 1.1867599487304688,
+      "rewards/rejected": -2.6845879554748535,
+      "step": 3880
+    },
+    {
+      "epoch": 0.5726483144413367,
+      "grad_norm": 57.025811267457534,
+      "learning_rate": 2.3029877059954414e-07,
+      "logits/chosen": -1.7329070568084717,
+      "logits/rejected": -1.3369855880737305,
+      "logps/chosen": -398.78497314453125,
+      "logps/rejected": -550.7677001953125,
+      "loss": 0.4763,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.4718331098556519,
+      "rewards/margins": 1.6102197170257568,
+      "rewards/rejected": -3.0820529460906982,
+      "step": 3890
+    },
+    {
+      "epoch": 0.5741204180774326,
+      "grad_norm": 106.08643711485834,
+      "learning_rate": 2.2901823218536693e-07,
+      "logits/chosen": -1.489159345626831,
+      "logits/rejected": -1.0795636177062988,
+      "logps/chosen": -426.4410705566406,
+      "logps/rejected": -435.78369140625,
+      "loss": 0.5212,
+      "rewards/accuracies": 0.5666666626930237,
+      "rewards/chosen": -1.8088414669036865,
+      "rewards/margins": 0.6170053482055664,
+      "rewards/rejected": -2.425846815109253,
+      "step": 3900
+    },
+    {
+      "epoch": 0.5755925217135286,
+      "grad_norm": 54.81465542460915,
+      "learning_rate": 2.2773824792723428e-07,
+      "logits/chosen": -1.1983683109283447,
+      "logits/rejected": -1.030940055847168,
+      "logps/chosen": -374.3813171386719,
+      "logps/rejected": -536.5989379882812,
+      "loss": 0.4032,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -1.4547827243804932,
+      "rewards/margins": 1.6514374017715454,
+      "rewards/rejected": -3.106220245361328,
+      "step": 3910
+    },
+    {
+      "epoch": 0.5770646253496247,
+      "grad_norm": 131.8046182513765,
+      "learning_rate": 2.2645885163121156e-07,
+      "logits/chosen": -1.4867804050445557,
+      "logits/rejected": -1.2189862728118896,
+      "logps/chosen": -443.75689697265625,
+      "logps/rejected": -578.314208984375,
+      "loss": 0.4408,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.6495990753173828,
+      "rewards/margins": 1.5253831148147583,
+      "rewards/rejected": -3.1749820709228516,
+      "step": 3920
+    },
+    {
+      "epoch": 0.5785367289857206,
+      "grad_norm": 88.47104333244829,
+      "learning_rate": 2.2518007708783512e-07,
+      "logits/chosen": -1.2885537147521973,
+      "logits/rejected": -0.8970157504081726,
+      "logps/chosen": -467.15289306640625,
+      "logps/rejected": -679.8993530273438,
+      "loss": 0.5568,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0828945636749268,
+      "rewards/margins": 1.580463171005249,
+      "rewards/rejected": -3.6633574962615967,
+      "step": 3930
+    },
+    {
+      "epoch": 0.5800088326218166,
+      "grad_norm": 144.53884960188145,
+      "learning_rate": 2.2390195807122027e-07,
+      "logits/chosen": -1.1762340068817139,
+      "logits/rejected": -1.1747605800628662,
+      "logps/chosen": -401.1009826660156,
+      "logps/rejected": -435.6044006347656,
+      "loss": 0.5344,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -1.4244767427444458,
+      "rewards/margins": 0.6796760559082031,
+      "rewards/rejected": -2.1041529178619385,
+      "step": 3940
+    },
+    {
+      "epoch": 0.5814809362579125,
+      "grad_norm": 97.47022453059363,
+      "learning_rate": 2.2262452833816886e-07,
+      "logits/chosen": -1.1835429668426514,
+      "logits/rejected": -0.4401678442955017,
+      "logps/chosen": -419.27301025390625,
+      "logps/rejected": -492.27447509765625,
+      "loss": 0.551,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.656942367553711,
+      "rewards/margins": 1.1906611919403076,
+      "rewards/rejected": -2.8476035594940186,
+      "step": 3950
+    },
+    {
+      "epoch": 0.5829530398940085,
+      "grad_norm": 76.22444315682189,
+      "learning_rate": 2.2134782162727778e-07,
+      "logits/chosen": -1.3269715309143066,
+      "logits/rejected": -1.03289794921875,
+      "logps/chosen": -398.5579528808594,
+      "logps/rejected": -494.4390563964844,
+      "loss": 0.546,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.5403027534484863,
+      "rewards/margins": 1.2021934986114502,
+      "rewards/rejected": -2.7424960136413574,
+      "step": 3960
+    },
+    {
+      "epoch": 0.5844251435301046,
+      "grad_norm": 91.33028683132093,
+      "learning_rate": 2.2007187165804822e-07,
+      "logits/chosen": -1.200577974319458,
+      "logits/rejected": -0.8154312968254089,
+      "logps/chosen": -441.831298828125,
+      "logps/rejected": -612.3999633789062,
+      "loss": 0.576,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.56230628490448,
+      "rewards/margins": 1.3954347372055054,
+      "rewards/rejected": -2.9577410221099854,
+      "step": 3970
+    },
+    {
+      "epoch": 0.5858972471662005,
+      "grad_norm": 120.67388338591378,
+      "learning_rate": 2.1879671212999437e-07,
+      "logits/chosen": -1.3850030899047852,
+      "logits/rejected": -1.1487380266189575,
+      "logps/chosen": -394.9458923339844,
+      "logps/rejected": -439.26837158203125,
+      "loss": 0.4756,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -1.2616503238677979,
+      "rewards/margins": 0.9496402740478516,
+      "rewards/rejected": -2.2112908363342285,
+      "step": 3980
+    },
+    {
+      "epoch": 0.5873693508022965,
+      "grad_norm": 64.13137302467265,
+      "learning_rate": 2.1752237672175433e-07,
+      "logits/chosen": -1.5979244709014893,
+      "logits/rejected": -1.2981550693511963,
+      "logps/chosen": -412.94976806640625,
+      "logps/rejected": -476.72186279296875,
+      "loss": 0.496,
+      "rewards/accuracies": 0.6333333849906921,
+      "rewards/chosen": -1.4475165605545044,
+      "rewards/margins": 0.658544659614563,
+      "rewards/rejected": -2.1060609817504883,
+      "step": 3990
+    },
+    {
+      "epoch": 0.5888414544383924,
+      "grad_norm": 88.69610580552754,
+      "learning_rate": 2.162488990901998e-07,
+      "logits/chosen": -1.286516785621643,
+      "logits/rejected": -1.230358362197876,
+      "logps/chosen": -462.65252685546875,
+      "logps/rejected": -565.0322265625,
+      "loss": 0.4958,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.040295124053955,
+      "rewards/margins": 1.1627228260040283,
+      "rewards/rejected": -3.2030181884765625,
+      "step": 4000
+    },
+    {
+      "epoch": 0.5903135580744885,
+      "grad_norm": 82.8018643796261,
+      "learning_rate": 2.1497631286954764e-07,
+      "logits/chosen": -1.5582008361816406,
+      "logits/rejected": -1.0771392583847046,
+      "logps/chosen": -487.6258850097656,
+      "logps/rejected": -536.5396728515625,
+      "loss": 0.4859,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.5401533842086792,
+      "rewards/margins": 1.3004016876220703,
+      "rewards/rejected": -2.840555191040039,
+      "step": 4010
+    },
+    {
+      "epoch": 0.5917856617105844,
+      "grad_norm": 164.80643813291522,
+      "learning_rate": 2.1370465167047118e-07,
+      "logits/chosen": -1.5033612251281738,
+      "logits/rejected": -1.1781336069107056,
+      "logps/chosen": -415.71392822265625,
+      "logps/rejected": -462.5985412597656,
+      "loss": 0.49,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7596426010131836,
+      "rewards/margins": 0.992581844329834,
+      "rewards/rejected": -2.7522244453430176,
+      "step": 4020
+    },
+    {
+      "epoch": 0.5932577653466804,
+      "grad_norm": 59.8446287734588,
+      "learning_rate": 2.124339490792128e-07,
+      "logits/chosen": -1.4557219743728638,
+      "logits/rejected": -1.3001980781555176,
+      "logps/chosen": -452.73126220703125,
+      "logps/rejected": -607.0549926757812,
+      "loss": 0.4293,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.842187523841858,
+      "rewards/margins": 1.4325052499771118,
+      "rewards/rejected": -3.2746925354003906,
+      "step": 4030
+    },
+    {
+      "epoch": 0.5947298689827764,
+      "grad_norm": 99.88325490179106,
+      "learning_rate": 2.1116423865669703e-07,
+      "logits/chosen": -1.559486746788025,
+      "logits/rejected": -0.9255777597427368,
+      "logps/chosen": -440.6299743652344,
+      "logps/rejected": -545.955078125,
+      "loss": 0.4867,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.7578004598617554,
+      "rewards/margins": 1.2535126209259033,
+      "rewards/rejected": -3.011312961578369,
+      "step": 4040
+    },
+    {
+      "epoch": 0.5962019726188724,
+      "grad_norm": 89.91731817292883,
+      "learning_rate": 2.0989555393764354e-07,
+      "logits/chosen": -1.293605923652649,
+      "logits/rejected": -1.3127516508102417,
+      "logps/chosen": -565.013671875,
+      "logps/rejected": -562.1390380859375,
+      "loss": 0.5946,
+      "rewards/accuracies": 0.5333333611488342,
+      "rewards/chosen": -1.9389889240264893,
+      "rewards/margins": 0.5274927616119385,
+      "rewards/rejected": -2.4664814472198486,
+      "step": 4050
+    },
+    {
+      "epoch": 0.5976740762549684,
+      "grad_norm": 75.61195520655684,
+      "learning_rate": 2.0862792842968214e-07,
+      "logits/chosen": -1.2534714937210083,
+      "logits/rejected": -0.6822981834411621,
+      "logps/chosen": -474.09326171875,
+      "logps/rejected": -471.49560546875,
+      "loss": 0.5203,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.6415497064590454,
+      "rewards/margins": 0.8943950533866882,
+      "rewards/rejected": -2.535945177078247,
+      "step": 4060
+    },
+    {
+      "epoch": 0.5991461798910643,
+      "grad_norm": 74.48009054349463,
+      "learning_rate": 2.0736139561246713e-07,
+      "logits/chosen": -1.2605535984039307,
+      "logits/rejected": -0.9497723579406738,
+      "logps/chosen": -429.6415100097656,
+      "logps/rejected": -536.9093627929688,
+      "loss": 0.5403,
+      "rewards/accuracies": 0.5999999642372131,
+      "rewards/chosen": -1.7722374200820923,
+      "rewards/margins": 0.9319728016853333,
+      "rewards/rejected": -2.704209804534912,
+      "step": 4070
+    },
+    {
+      "epoch": 0.6006182835271603,
+      "grad_norm": 167.6179453167691,
+      "learning_rate": 2.060959889367938e-07,
+      "logits/chosen": -1.4208152294158936,
+      "logits/rejected": -1.0713694095611572,
+      "logps/chosen": -444.0927734375,
+      "logps/rejected": -618.0239868164062,
+      "loss": 0.5208,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": -1.594525694847107,
+      "rewards/margins": 1.8230946063995361,
+      "rewards/rejected": -3.4176204204559326,
+      "step": 4080
+    },
+    {
+      "epoch": 0.6020903871632562,
+      "grad_norm": 58.42146368479048,
+      "learning_rate": 2.0483174182371435e-07,
+      "logits/chosen": -1.2485980987548828,
+      "logits/rejected": -1.230147123336792,
+      "logps/chosen": -447.17877197265625,
+      "logps/rejected": -491.56512451171875,
+      "loss": 0.5198,
+      "rewards/accuracies": 0.4999999403953552,
+      "rewards/chosen": -1.8198697566986084,
+      "rewards/margins": 0.40783554315567017,
+      "rewards/rejected": -2.227705478668213,
+      "step": 4090
+    },
+    {
+      "epoch": 0.6035624907993523,
+      "grad_norm": 77.7073924106762,
+      "learning_rate": 2.0356868766365536e-07,
+      "logits/chosen": -1.4538729190826416,
+      "logits/rejected": -1.2259035110473633,
+      "logps/chosen": -473.74658203125,
+      "logps/rejected": -567.4500732421875,
+      "loss": 0.5613,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.8328685760498047,
+      "rewards/margins": 0.5852871537208557,
+      "rewards/rejected": -2.4181556701660156,
+      "step": 4100
+    },
+    {
+      "epoch": 0.6050345944354483,
+      "grad_norm": 47.20760639602046,
+      "learning_rate": 2.023068598155363e-07,
+      "logits/chosen": -1.5401443243026733,
+      "logits/rejected": -1.2264618873596191,
+      "logps/chosen": -512.0546264648438,
+      "logps/rejected": -492.6022033691406,
+      "loss": 0.5921,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.111586809158325,
+      "rewards/margins": 0.5268180966377258,
+      "rewards/rejected": -2.638404369354248,
+      "step": 4110
+    },
+    {
+      "epoch": 0.6065066980715442,
+      "grad_norm": 166.3540160109531,
+      "learning_rate": 2.010462916058875e-07,
+      "logits/chosen": -1.5501317977905273,
+      "logits/rejected": -1.3588258028030396,
+      "logps/chosen": -495.94903564453125,
+      "logps/rejected": -630.6212158203125,
+      "loss": 0.4753,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -1.701025366783142,
+      "rewards/margins": 1.473243236541748,
+      "rewards/rejected": -3.174268960952759,
+      "step": 4120
+    },
+    {
+      "epoch": 0.6079788017076402,
+      "grad_norm": 76.88460088192504,
+      "learning_rate": 1.9978701632797118e-07,
+      "logits/chosen": -1.4404414892196655,
+      "logits/rejected": -1.3681303262710571,
+      "logps/chosen": -391.15899658203125,
+      "logps/rejected": -478.22723388671875,
+      "loss": 0.4961,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -1.62236750125885,
+      "rewards/margins": 0.7691271901130676,
+      "rewards/rejected": -2.3914947509765625,
+      "step": 4130
+    },
+    {
+      "epoch": 0.6094509053437362,
+      "grad_norm": 78.04768322916944,
+      "learning_rate": 1.9852906724090127e-07,
+      "logits/chosen": -1.5495954751968384,
+      "logits/rejected": -1.3348801136016846,
+      "logps/chosen": -378.16619873046875,
+      "logps/rejected": -523.1204223632812,
+      "loss": 0.4102,
+      "rewards/accuracies": 0.73333340883255,
+      "rewards/chosen": -1.6241337060928345,
+      "rewards/margins": 1.2306989431381226,
+      "rewards/rejected": -2.854832410812378,
+      "step": 4140
+    },
+    {
+      "epoch": 0.6109230089798322,
+      "grad_norm": 71.77760045629627,
+      "learning_rate": 1.9727247756876534e-07,
+      "logits/chosen": -1.3857651948928833,
+      "logits/rejected": -1.4891043901443481,
+      "logps/chosen": -376.3533020019531,
+      "logps/rejected": -423.8584899902344,
+      "loss": 0.5789,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.4472265243530273,
+      "rewards/margins": 0.23889613151550293,
+      "rewards/rejected": -1.6861225366592407,
+      "step": 4150
+    },
+    {
+      "epoch": 0.6123951126159282,
+      "grad_norm": 154.44237844465218,
+      "learning_rate": 1.9601728049974683e-07,
+      "logits/chosen": -1.0164905786514282,
+      "logits/rejected": -0.9828926920890808,
+      "logps/chosen": -404.97601318359375,
+      "logps/rejected": -601.942626953125,
+      "loss": 0.4779,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.8876426219940186,
+      "rewards/margins": 1.514784574508667,
+      "rewards/rejected": -3.4024271965026855,
+      "step": 4160
+    },
+    {
+      "epoch": 0.6138672162520241,
+      "grad_norm": 141.1191225899058,
+      "learning_rate": 1.94763509185249e-07,
+      "logits/chosen": -1.2642747163772583,
+      "logits/rejected": -0.9743534922599792,
+      "logps/chosen": -449.161865234375,
+      "logps/rejected": -586.4234619140625,
+      "loss": 0.4459,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.940500020980835,
+      "rewards/margins": 1.268189787864685,
+      "rewards/rejected": -3.2086899280548096,
+      "step": 4170
+    },
+    {
+      "epoch": 0.6153393198881201,
+      "grad_norm": 94.6393683561518,
+      "learning_rate": 1.935111967390189e-07,
+      "logits/chosen": -1.464521050453186,
+      "logits/rejected": -0.9969412088394165,
+      "logps/chosen": -416.349365234375,
+      "logps/rejected": -506.1065368652344,
+      "loss": 0.4172,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.7117770910263062,
+      "rewards/margins": 1.4608806371688843,
+      "rewards/rejected": -3.1726577281951904,
+      "step": 4180
+    },
+    {
+      "epoch": 0.6168114235242161,
+      "grad_norm": 144.02589303977794,
+      "learning_rate": 1.922603762362729e-07,
+      "logits/chosen": -1.304671049118042,
+      "logits/rejected": -0.6768732070922852,
+      "logps/chosen": -444.5458984375,
+      "logps/rejected": -567.3345947265625,
+      "loss": 0.5234,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1895623207092285,
+      "rewards/margins": 1.1984633207321167,
+      "rewards/rejected": -3.3880257606506348,
+      "step": 4190
+    },
+    {
+      "epoch": 0.6182835271603121,
+      "grad_norm": 152.47598774250133,
+      "learning_rate": 1.9101108071282342e-07,
+      "logits/chosen": -1.3892663717269897,
+      "logits/rejected": -0.9998384714126587,
+      "logps/chosen": -502.8368225097656,
+      "logps/rejected": -621.2618408203125,
+      "loss": 0.4721,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.2931313514709473,
+      "rewards/margins": 1.647463083267212,
+      "rewards/rejected": -3.94059419631958,
+      "step": 4200
+    },
+    {
+      "epoch": 0.619755630796408,
+      "grad_norm": 209.9216993920297,
+      "learning_rate": 1.8976334316420576e-07,
+      "logits/chosen": -1.6642833948135376,
+      "logits/rejected": -1.2484536170959473,
+      "logps/chosen": -546.4644165039062,
+      "logps/rejected": -595.1619262695312,
+      "loss": 0.4791,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.100569009780884,
+      "rewards/margins": 1.0795539617538452,
+      "rewards/rejected": -3.1801230907440186,
+      "step": 4210
+    },
+    {
+      "epoch": 0.621227734432504,
+      "grad_norm": 94.85024625133835,
+      "learning_rate": 1.8851719654480748e-07,
+      "logits/chosen": -1.5108282566070557,
+      "logits/rejected": -1.0989423990249634,
+      "logps/chosen": -447.67327880859375,
+      "logps/rejected": -557.6844482421875,
+      "loss": 0.4872,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.9102060794830322,
+      "rewards/margins": 1.181675910949707,
+      "rewards/rejected": -3.0918819904327393,
+      "step": 4220
+    },
+    {
+      "epoch": 0.6226998380686001,
+      "grad_norm": 144.09810984087537,
+      "learning_rate": 1.8727267376699735e-07,
+      "logits/chosen": -1.419729471206665,
+      "logits/rejected": -0.9745621681213379,
+      "logps/chosen": -510.80047607421875,
+      "logps/rejected": -543.3255615234375,
+      "loss": 0.5549,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.880112886428833,
+      "rewards/margins": 1.0943782329559326,
+      "rewards/rejected": -2.9744911193847656,
+      "step": 4230
+    },
+    {
+      "epoch": 0.624171941704696,
+      "grad_norm": 65.32185352345809,
+      "learning_rate": 1.8602980770025645e-07,
+      "logits/chosen": -1.2146351337432861,
+      "logits/rejected": -0.7923814058303833,
+      "logps/chosen": -451.31658935546875,
+      "logps/rejected": -515.2090454101562,
+      "loss": 0.4314,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.5133302211761475,
+      "rewards/margins": 1.4617403745651245,
+      "rewards/rejected": -2.9750709533691406,
+      "step": 4240
+    },
+    {
+      "epoch": 0.625644045340792,
+      "grad_norm": 400.9336865441555,
+      "learning_rate": 1.8478863117031007e-07,
+      "logits/chosen": -1.047730565071106,
+      "logits/rejected": -0.43196743726730347,
+      "logps/chosen": -457.43035888671875,
+      "logps/rejected": -635.474609375,
+      "loss": 0.5508,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -2.315809488296509,
+      "rewards/margins": 1.5847208499908447,
+      "rewards/rejected": -3.9005303382873535,
+      "step": 4250
+    },
+    {
+      "epoch": 0.6271161489768879,
+      "grad_norm": 190.76353636910375,
+      "learning_rate": 1.8354917695826026e-07,
+      "logits/chosen": -1.206991195678711,
+      "logits/rejected": -1.0310016870498657,
+      "logps/chosen": -580.2699584960938,
+      "logps/rejected": -672.878662109375,
+      "loss": 0.4824,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.416865825653076,
+      "rewards/margins": 1.251690149307251,
+      "rewards/rejected": -3.668555736541748,
+      "step": 4260
+    },
+    {
+      "epoch": 0.6285882526129839,
+      "grad_norm": 68.9692743255753,
+      "learning_rate": 1.8231147779972074e-07,
+      "logits/chosen": -0.866108238697052,
+      "logits/rejected": -0.6675370931625366,
+      "logps/chosen": -403.5827331542969,
+      "logps/rejected": -582.376953125,
+      "loss": 0.5206,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.99154531955719,
+      "rewards/margins": 1.4701696634292603,
+      "rewards/rejected": -3.46171498298645,
+      "step": 4270
+    },
+    {
+      "epoch": 0.63006035624908,
+      "grad_norm": 44.600304662020214,
+      "learning_rate": 1.8107556638395168e-07,
+      "logits/chosen": -1.6005884408950806,
+      "logits/rejected": -1.0544960498809814,
+      "logps/chosen": -445.9622497558594,
+      "logps/rejected": -620.0938720703125,
+      "loss": 0.4594,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.6346498727798462,
+      "rewards/margins": 1.7904284000396729,
+      "rewards/rejected": -3.4250781536102295,
+      "step": 4280
+    },
+    {
+      "epoch": 0.6315324598851759,
+      "grad_norm": 125.88691549754662,
+      "learning_rate": 1.798414753529971e-07,
+      "logits/chosen": -1.414941668510437,
+      "logits/rejected": -1.3350245952606201,
+      "logps/chosen": -474.35955810546875,
+      "logps/rejected": -544.1416625976562,
+      "loss": 0.4221,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.7634519338607788,
+      "rewards/margins": 1.0667226314544678,
+      "rewards/rejected": -2.830174684524536,
+      "step": 4290
+    },
+    {
+      "epoch": 0.6330045635212719,
+      "grad_norm": 95.99487386342098,
+      "learning_rate": 1.7860923730082152e-07,
+      "logits/chosen": -1.4767735004425049,
+      "logits/rejected": -1.374411940574646,
+      "logps/chosen": -537.554443359375,
+      "logps/rejected": -551.3592529296875,
+      "loss": 0.5782,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.1924350261688232,
+      "rewards/margins": 0.7634781002998352,
+      "rewards/rejected": -2.9559130668640137,
+      "step": 4300
+    },
+    {
+      "epoch": 0.6344766671573678,
+      "grad_norm": 62.96501205963991,
+      "learning_rate": 1.7737888477245052e-07,
+      "logits/chosen": -1.388330101966858,
+      "logits/rejected": -1.2299821376800537,
+      "logps/chosen": -537.3853149414062,
+      "logps/rejected": -637.923095703125,
+      "loss": 0.4149,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.465048313140869,
+      "rewards/margins": 1.0545275211334229,
+      "rewards/rejected": -3.519575834274292,
+      "step": 4310
+    },
+    {
+      "epoch": 0.6359487707934639,
+      "grad_norm": 141.6486197599099,
+      "learning_rate": 1.761504502631102e-07,
+      "logits/chosen": -1.5358697175979614,
+      "logits/rejected": -1.291987657546997,
+      "logps/chosen": -504.5132751464844,
+      "logps/rejected": -681.7574462890625,
+      "loss": 0.5659,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -2.266136646270752,
+      "rewards/margins": 1.7751989364624023,
+      "rewards/rejected": -4.041335582733154,
+      "step": 4320
+    },
+    {
+      "epoch": 0.6374208744295599,
+      "grad_norm": 175.4677207040497,
+      "learning_rate": 1.749239662173693e-07,
+      "logits/chosen": -1.0412142276763916,
+      "logits/rejected": -0.8617256283760071,
+      "logps/chosen": -406.6339111328125,
+      "logps/rejected": -548.8243408203125,
+      "loss": 0.4149,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.2163126468658447,
+      "rewards/margins": 1.2508987188339233,
+      "rewards/rejected": -3.4672112464904785,
+      "step": 4330
+    },
+    {
+      "epoch": 0.6388929780656558,
+      "grad_norm": 136.91063087882577,
+      "learning_rate": 1.7369946502828245e-07,
+      "logits/chosen": -1.4333940744400024,
+      "logits/rejected": -1.0822374820709229,
+      "logps/chosen": -430.1373596191406,
+      "logps/rejected": -479.7125549316406,
+      "loss": 0.4464,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.5607531070709229,
+      "rewards/margins": 1.2057816982269287,
+      "rewards/rejected": -2.7665348052978516,
+      "step": 4340
+    },
+    {
+      "epoch": 0.6403650817017518,
+      "grad_norm": 132.33571087083718,
+      "learning_rate": 1.7247697903653395e-07,
+      "logits/chosen": -1.5349242687225342,
+      "logits/rejected": -1.3145502805709839,
+      "logps/chosen": -502.62567138671875,
+      "logps/rejected": -641.2630004882812,
+      "loss": 0.4946,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8769954442977905,
+      "rewards/margins": 1.7127166986465454,
+      "rewards/rejected": -3.589712619781494,
+      "step": 4350
+    },
+    {
+      "epoch": 0.6418371853378478,
+      "grad_norm": 172.1367132744898,
+      "learning_rate": 1.7125654052958465e-07,
+      "logits/chosen": -1.3708072900772095,
+      "logits/rejected": -0.9546362161636353,
+      "logps/chosen": -458.1553649902344,
+      "logps/rejected": -583.6203002929688,
+      "loss": 0.4935,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.1133432388305664,
+      "rewards/margins": 1.4481130838394165,
+      "rewards/rejected": -3.5614559650421143,
+      "step": 4360
+    },
+    {
+      "epoch": 0.6433092889739438,
+      "grad_norm": 128.43439548785636,
+      "learning_rate": 1.7003818174081832e-07,
+      "logits/chosen": -1.554459810256958,
+      "logits/rejected": -0.766741156578064,
+      "logps/chosen": -456.7364196777344,
+      "logps/rejected": -559.1473388671875,
+      "loss": 0.4098,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.1493165493011475,
+      "rewards/margins": 1.483247995376587,
+      "rewards/rejected": -3.6325645446777344,
+      "step": 4370
+    },
+    {
+      "epoch": 0.6447813926100397,
+      "grad_norm": 164.25367557447944,
+      "learning_rate": 1.6882193484869067e-07,
+      "logits/chosen": -1.3203048706054688,
+      "logits/rejected": -1.0798929929733276,
+      "logps/chosen": -456.9297790527344,
+      "logps/rejected": -523.0709228515625,
+      "loss": 0.6116,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.040025234222412,
+      "rewards/margins": 0.8116199374198914,
+      "rewards/rejected": -2.851644992828369,
+      "step": 4380
+    },
+    {
+      "epoch": 0.6462534962461357,
+      "grad_norm": 106.43889028279479,
+      "learning_rate": 1.676078319758796e-07,
+      "logits/chosen": -1.5613614320755005,
+      "logits/rejected": -1.1758073568344116,
+      "logps/chosen": -428.8338317871094,
+      "logps/rejected": -505.5704040527344,
+      "loss": 0.4929,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.6768068075180054,
+      "rewards/margins": 1.3281102180480957,
+      "rewards/rejected": -3.0049169063568115,
+      "step": 4390
+    },
+    {
+      "epoch": 0.6477255998822317,
+      "grad_norm": 77.6104492984931,
+      "learning_rate": 1.6639590518843643e-07,
+      "logits/chosen": -1.5984054803848267,
+      "logits/rejected": -1.2882667779922485,
+      "logps/chosen": -479.27886962890625,
+      "logps/rejected": -680.8609619140625,
+      "loss": 0.4882,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -1.5643619298934937,
+      "rewards/margins": 1.9690698385238647,
+      "rewards/rejected": -3.5334315299987793,
+      "step": 4400
+    },
+    {
+      "epoch": 0.6491977035183277,
+      "grad_norm": 148.22327706014093,
+      "learning_rate": 1.6518618649493932e-07,
+      "logits/chosen": -1.2908194065093994,
+      "logits/rejected": -1.1842042207717896,
+      "logps/chosen": -464.754150390625,
+      "logps/rejected": -641.6333618164062,
+      "loss": 0.4216,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -2.0775306224823,
+      "rewards/margins": 1.6497796773910522,
+      "rewards/rejected": -3.7273101806640625,
+      "step": 4410
+    },
+    {
+      "epoch": 0.6506698071544237,
+      "grad_norm": 90.50992151430243,
+      "learning_rate": 1.6397870784564777e-07,
+      "logits/chosen": -1.3347129821777344,
+      "logits/rejected": -1.1741870641708374,
+      "logps/chosen": -357.79046630859375,
+      "logps/rejected": -620.353271484375,
+      "loss": 0.5393,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.4308732748031616,
+      "rewards/margins": 1.9854919910430908,
+      "rewards/rejected": -3.416365385055542,
+      "step": 4420
+    },
+    {
+      "epoch": 0.6521419107905196,
+      "grad_norm": 133.53822255398748,
+      "learning_rate": 1.6277350113165887e-07,
+      "logits/chosen": -1.4606850147247314,
+      "logits/rejected": -1.0468692779541016,
+      "logps/chosen": -447.333251953125,
+      "logps/rejected": -506.942626953125,
+      "loss": 0.5451,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.8336231708526611,
+      "rewards/margins": 1.1235220432281494,
+      "rewards/rejected": -2.9571454524993896,
+      "step": 4430
+    },
+    {
+      "epoch": 0.6536140144266156,
+      "grad_norm": 85.62298259101672,
+      "learning_rate": 1.615705981840646e-07,
+      "logits/chosen": -1.5510303974151611,
+      "logits/rejected": -1.2544745206832886,
+      "logps/chosen": -500.2935485839844,
+      "logps/rejected": -588.1988525390625,
+      "loss": 0.5077,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -1.784166693687439,
+      "rewards/margins": 1.1019845008850098,
+      "rewards/rejected": -2.8861513137817383,
+      "step": 4440
+    },
+    {
+      "epoch": 0.6550861180627117,
+      "grad_norm": 95.86145688771262,
+      "learning_rate": 1.6037003077311178e-07,
+      "logits/chosen": -1.5594561100006104,
+      "logits/rejected": -1.3050320148468018,
+      "logps/chosen": -387.09991455078125,
+      "logps/rejected": -573.3997802734375,
+      "loss": 0.5525,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.4168446063995361,
+      "rewards/margins": 1.6031293869018555,
+      "rewards/rejected": -3.0199742317199707,
+      "step": 4450
+    },
+    {
+      "epoch": 0.6565582216988076,
+      "grad_norm": 49.462407570429335,
+      "learning_rate": 1.591718306073625e-07,
+      "logits/chosen": -1.7461349964141846,
+      "logits/rejected": -1.0915542840957642,
+      "logps/chosen": -622.7125854492188,
+      "logps/rejected": -599.0344848632812,
+      "loss": 0.4926,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.8506015539169312,
+      "rewards/margins": 1.1055147647857666,
+      "rewards/rejected": -2.956116199493408,
+      "step": 4460
+    },
+    {
+      "epoch": 0.6580303253349036,
+      "grad_norm": 72.30120577201066,
+      "learning_rate": 1.5797602933285672e-07,
+      "logits/chosen": -1.4067375659942627,
+      "logits/rejected": -0.7826912999153137,
+      "logps/chosen": -453.6651916503906,
+      "logps/rejected": -578.394287109375,
+      "loss": 0.4545,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.7367359399795532,
+      "rewards/margins": 1.6170374155044556,
+      "rewards/rejected": -3.3537731170654297,
+      "step": 4470
+    },
+    {
+      "epoch": 0.6595024289709995,
+      "grad_norm": 103.57322200861036,
+      "learning_rate": 1.5678265853227679e-07,
+      "logits/chosen": -1.2267433404922485,
+      "logits/rejected": -1.1783136129379272,
+      "logps/chosen": -474.3990173339844,
+      "logps/rejected": -685.5614013671875,
+      "loss": 0.4714,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.777988076210022,
+      "rewards/margins": 1.7630653381347656,
+      "rewards/rejected": -3.541053295135498,
+      "step": 4480
+    },
+    {
+      "epoch": 0.6609745326070955,
+      "grad_norm": 191.72471270365537,
+      "learning_rate": 1.5559174972411271e-07,
+      "logits/chosen": -1.4727813005447388,
+      "logits/rejected": -0.9439153671264648,
+      "logps/chosen": -529.7391357421875,
+      "logps/rejected": -677.0784912109375,
+      "loss": 0.4304,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.958915114402771,
+      "rewards/margins": 1.8345258235931396,
+      "rewards/rejected": -3.793440341949463,
+      "step": 4490
+    },
+    {
+      "epoch": 0.6624466362431916,
+      "grad_norm": 172.11356470969972,
+      "learning_rate": 1.5440333436183022e-07,
+      "logits/chosen": -1.05788254737854,
+      "logits/rejected": -0.7720486521720886,
+      "logps/chosen": -381.38128662109375,
+      "logps/rejected": -580.8397216796875,
+      "loss": 0.506,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.575581669807434,
+      "rewards/margins": 2.198709011077881,
+      "rewards/rejected": -3.7742908000946045,
+      "step": 4500
+    },
+    {
+      "epoch": 0.6639187398792875,
+      "grad_norm": 67.77998755873953,
+      "learning_rate": 1.532174438330399e-07,
+      "logits/chosen": -1.2263076305389404,
+      "logits/rejected": -0.8507343530654907,
+      "logps/chosen": -513.4577026367188,
+      "logps/rejected": -536.4158325195312,
+      "loss": 0.3991,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8174861669540405,
+      "rewards/margins": 1.1229370832443237,
+      "rewards/rejected": -2.9404232501983643,
+      "step": 4510
+    },
+    {
+      "epoch": 0.6653908435153835,
+      "grad_norm": 64.31113907340405,
+      "learning_rate": 1.5203410945866807e-07,
+      "logits/chosen": -1.3456099033355713,
+      "logits/rejected": -1.0697879791259766,
+      "logps/chosen": -442.7084045410156,
+      "logps/rejected": -588.1572875976562,
+      "loss": 0.4759,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.9251432418823242,
+      "rewards/margins": 1.2387527227401733,
+      "rewards/rejected": -3.163896322250366,
+      "step": 4520
+    },
+    {
+      "epoch": 0.6668629471514794,
+      "grad_norm": 171.34155081331926,
+      "learning_rate": 1.5085336249212982e-07,
+      "logits/chosen": -0.9265859723091125,
+      "logits/rejected": -0.7449873685836792,
+      "logps/chosen": -496.591796875,
+      "logps/rejected": -637.4634399414062,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.263828754425049,
+      "rewards/margins": 1.768465280532837,
+      "rewards/rejected": -4.032293319702148,
+      "step": 4530
+    },
+    {
+      "epoch": 0.6683350507875755,
+      "grad_norm": 65.3982082721759,
+      "learning_rate": 1.4967523411850314e-07,
+      "logits/chosen": -1.4070160388946533,
+      "logits/rejected": -1.2556936740875244,
+      "logps/chosen": -415.35693359375,
+      "logps/rejected": -572.4760131835938,
+      "loss": 0.4425,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.8644262552261353,
+      "rewards/margins": 1.3102327585220337,
+      "rewards/rejected": -3.174659013748169,
+      "step": 4540
+    },
+    {
+      "epoch": 0.6698071544236714,
+      "grad_norm": 86.57165747995961,
+      "learning_rate": 1.484997554537057e-07,
+      "logits/chosen": -1.4679739475250244,
+      "logits/rejected": -1.3335583209991455,
+      "logps/chosen": -442.01025390625,
+      "logps/rejected": -588.5447998046875,
+      "loss": 0.5407,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -1.6064382791519165,
+      "rewards/margins": 1.1679922342300415,
+      "rewards/rejected": -2.774430751800537,
+      "step": 4550
+    },
+    {
+      "epoch": 0.6712792580597674,
+      "grad_norm": 169.8394168938232,
+      "learning_rate": 1.4732695754367287e-07,
+      "logits/chosen": -1.4626400470733643,
+      "logits/rejected": -1.1115871667861938,
+      "logps/chosen": -470.23712158203125,
+      "logps/rejected": -604.0081787109375,
+      "loss": 0.5613,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.7982364892959595,
+      "rewards/margins": 1.3922786712646484,
+      "rewards/rejected": -3.1905152797698975,
+      "step": 4560
+    },
+    {
+      "epoch": 0.6727513616958634,
+      "grad_norm": 118.94477345887874,
+      "learning_rate": 1.4615687136353787e-07,
+      "logits/chosen": -1.449825644493103,
+      "logits/rejected": -0.9356663823127747,
+      "logps/chosen": -477.78485107421875,
+      "logps/rejected": -499.4154357910156,
+      "loss": 0.5569,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -1.8658859729766846,
+      "rewards/margins": 0.924848735332489,
+      "rewards/rejected": -2.7907345294952393,
+      "step": 4570
+    },
+    {
+      "epoch": 0.6742234653319593,
+      "grad_norm": 114.04061458733868,
+      "learning_rate": 1.4498952781681328e-07,
+      "logits/chosen": -1.372307300567627,
+      "logits/rejected": -1.2247451543807983,
+      "logps/chosen": -397.2774963378906,
+      "logps/rejected": -504.745849609375,
+      "loss": 0.4547,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.627138376235962,
+      "rewards/margins": 1.314136266708374,
+      "rewards/rejected": -2.941274881362915,
+      "step": 4580
+    },
+    {
+      "epoch": 0.6756955689680554,
+      "grad_norm": 96.84533699255098,
+      "learning_rate": 1.4382495773457544e-07,
+      "logits/chosen": -1.3611235618591309,
+      "logits/rejected": -1.2581226825714111,
+      "logps/chosen": -409.7996520996094,
+      "logps/rejected": -437.6298828125,
+      "loss": 0.6343,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.4521945714950562,
+      "rewards/margins": 0.7812715768814087,
+      "rewards/rejected": -2.233466148376465,
+      "step": 4590
+    },
+    {
+      "epoch": 0.6771676726041513,
+      "grad_norm": 65.63651763622458,
+      "learning_rate": 1.4266319187464965e-07,
+      "logits/chosen": -1.4840503931045532,
+      "logits/rejected": -1.2432677745819092,
+      "logps/chosen": -336.3072204589844,
+      "logps/rejected": -436.96893310546875,
+      "loss": 0.4133,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.274876356124878,
+      "rewards/margins": 1.0072202682495117,
+      "rewards/rejected": -2.2820963859558105,
+      "step": 4600
+    },
+    {
+      "epoch": 0.6786397762402473,
+      "grad_norm": 93.43058578636344,
+      "learning_rate": 1.415042609207981e-07,
+      "logits/chosen": -1.6027921438217163,
+      "logits/rejected": -1.5193991661071777,
+      "logps/chosen": -348.9036865234375,
+      "logps/rejected": -416.65673828125,
+      "loss": 0.451,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.2478983402252197,
+      "rewards/margins": 1.0321069955825806,
+      "rewards/rejected": -2.28000545501709,
+      "step": 4610
+    },
+    {
+      "epoch": 0.6801118798763433,
+      "grad_norm": 273.7130929806925,
+      "learning_rate": 1.4034819548190936e-07,
+      "logits/chosen": -1.3875776529312134,
+      "logits/rejected": -1.041291356086731,
+      "logps/chosen": -430.201904296875,
+      "logps/rejected": -521.618896484375,
+      "loss": 0.5041,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.8789126873016357,
+      "rewards/margins": 1.1863256692886353,
+      "rewards/rejected": -3.0652387142181396,
+      "step": 4620
+    },
+    {
+      "epoch": 0.6815839835124393,
+      "grad_norm": 83.32445019454997,
+      "learning_rate": 1.3919502609119004e-07,
+      "logits/chosen": -1.5672070980072021,
+      "logits/rejected": -1.4558014869689941,
+      "logps/chosen": -503.7395935058594,
+      "logps/rejected": -586.6051025390625,
+      "loss": 0.5239,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.074951171875,
+      "rewards/margins": 0.9296423196792603,
+      "rewards/rejected": -3.0045933723449707,
+      "step": 4630
+    },
+    {
+      "epoch": 0.6830560871485353,
+      "grad_norm": 363.9267674425008,
+      "learning_rate": 1.380447832053583e-07,
+      "logits/chosen": -1.6036059856414795,
+      "logits/rejected": -1.3291890621185303,
+      "logps/chosen": -404.474365234375,
+      "logps/rejected": -542.76123046875,
+      "loss": 0.5525,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.660895586013794,
+      "rewards/margins": 1.3934032917022705,
+      "rewards/rejected": -3.0542988777160645,
+      "step": 4640
+    },
+    {
+      "epoch": 0.6845281907846312,
+      "grad_norm": 92.33024325861862,
+      "learning_rate": 1.3689749720383934e-07,
+      "logits/chosen": -1.6047083139419556,
+      "logits/rejected": -1.4916231632232666,
+      "logps/chosen": -397.97052001953125,
+      "logps/rejected": -433.46014404296875,
+      "loss": 0.4918,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": -1.2873114347457886,
+      "rewards/margins": 0.6466048359870911,
+      "rewards/rejected": -1.9339163303375244,
+      "step": 4650
+    },
+    {
+      "epoch": 0.6860002944207272,
+      "grad_norm": 210.9596609081259,
+      "learning_rate": 1.357531983879633e-07,
+      "logits/chosen": -1.475263237953186,
+      "logits/rejected": -1.409371256828308,
+      "logps/chosen": -379.2502136230469,
+      "logps/rejected": -444.80926513671875,
+      "loss": 0.5569,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.7458136081695557,
+      "rewards/margins": 0.8153721690177917,
+      "rewards/rejected": -2.561185359954834,
+      "step": 4660
+    },
+    {
+      "epoch": 0.6874723980568233,
+      "grad_norm": 76.12901361111314,
+      "learning_rate": 1.3461191698016482e-07,
+      "logits/chosen": -1.341385841369629,
+      "logits/rejected": -1.0819720029830933,
+      "logps/chosen": -445.7750549316406,
+      "logps/rejected": -526.4230346679688,
+      "loss": 0.4708,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8485463857650757,
+      "rewards/margins": 1.1189892292022705,
+      "rewards/rejected": -2.9675354957580566,
+      "step": 4670
+    },
+    {
+      "epoch": 0.6889445016929192,
+      "grad_norm": 100.19094061054672,
+      "learning_rate": 1.3347368312318475e-07,
+      "logits/chosen": -1.662327527999878,
+      "logits/rejected": -1.525238275527954,
+      "logps/chosen": -466.75286865234375,
+      "logps/rejected": -534.04541015625,
+      "loss": 0.5664,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.3829591274261475,
+      "rewards/margins": 0.8820352554321289,
+      "rewards/rejected": -2.2649941444396973,
+      "step": 4680
+    },
+    {
+      "epoch": 0.6904166053290152,
+      "grad_norm": 166.55294478922897,
+      "learning_rate": 1.3233852687927415e-07,
+      "logits/chosen": -1.6528041362762451,
+      "logits/rejected": -1.4121334552764893,
+      "logps/chosen": -475.67962646484375,
+      "logps/rejected": -570.8133544921875,
+      "loss": 0.5363,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.9025332927703857,
+      "rewards/margins": 0.821917712688446,
+      "rewards/rejected": -2.7244510650634766,
+      "step": 4690
+    },
+    {
+      "epoch": 0.6918887089651111,
+      "grad_norm": 287.29458389480885,
+      "learning_rate": 1.3120647822940035e-07,
+      "logits/chosen": -1.6049671173095703,
+      "logits/rejected": -1.3347653150558472,
+      "logps/chosen": -402.11944580078125,
+      "logps/rejected": -521.8661499023438,
+      "loss": 0.4867,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.3917683362960815,
+      "rewards/margins": 1.5385221242904663,
+      "rewards/rejected": -2.930290460586548,
+      "step": 4700
+    },
+    {
+      "epoch": 0.6933608126012071,
+      "grad_norm": 101.51018229901935,
+      "learning_rate": 1.3007756707245488e-07,
+      "logits/chosen": -1.4593899250030518,
+      "logits/rejected": -1.464545488357544,
+      "logps/chosen": -394.59124755859375,
+      "logps/rejected": -506.69622802734375,
+      "loss": 0.5705,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.6084076166152954,
+      "rewards/margins": 1.1785833835601807,
+      "rewards/rejected": -2.7869908809661865,
+      "step": 4710
+    },
+    {
+      "epoch": 0.6948329162373031,
+      "grad_norm": 117.5415318646873,
+      "learning_rate": 1.2895182322446415e-07,
+      "logits/chosen": -1.433753252029419,
+      "logits/rejected": -1.2077696323394775,
+      "logps/chosen": -442.948974609375,
+      "logps/rejected": -500.4745178222656,
+      "loss": 0.4716,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.609473466873169,
+      "rewards/margins": 1.194166660308838,
+      "rewards/rejected": -2.803640127182007,
+      "step": 4720
+    },
+    {
+      "epoch": 0.6963050198733991,
+      "grad_norm": 122.46337705322347,
+      "learning_rate": 1.2782927641780167e-07,
+      "logits/chosen": -1.6359031200408936,
+      "logits/rejected": -1.5747005939483643,
+      "logps/chosen": -489.8575744628906,
+      "logps/rejected": -603.5863037109375,
+      "loss": 0.5277,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.7673225402832031,
+      "rewards/margins": 1.133762001991272,
+      "rewards/rejected": -2.9010844230651855,
+      "step": 4730
+    },
+    {
+      "epoch": 0.6977771235094951,
+      "grad_norm": 151.42316962916956,
+      "learning_rate": 1.2670995630040288e-07,
+      "logits/chosen": -1.5032744407653809,
+      "logits/rejected": -1.2521288394927979,
+      "logps/chosen": -407.8828430175781,
+      "logps/rejected": -499.286865234375,
+      "loss": 0.5647,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.746633529663086,
+      "rewards/margins": 1.0992298126220703,
+      "rewards/rejected": -2.8458633422851562,
+      "step": 4740
+    },
+    {
+      "epoch": 0.699249227145591,
+      "grad_norm": 61.373076329510646,
+      "learning_rate": 1.2559389243498213e-07,
+      "logits/chosen": -1.3075616359710693,
+      "logits/rejected": -1.3392479419708252,
+      "logps/chosen": -486.76788330078125,
+      "logps/rejected": -608.127197265625,
+      "loss": 0.4639,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.6814171075820923,
+      "rewards/margins": 1.5302196741104126,
+      "rewards/rejected": -3.211636781692505,
+      "step": 4750
+    },
+    {
+      "epoch": 0.7007213307816871,
+      "grad_norm": 102.80659546722374,
+      "learning_rate": 1.2448111429825198e-07,
+      "logits/chosen": -1.4003568887710571,
+      "logits/rejected": -1.1918909549713135,
+      "logps/chosen": -527.3383178710938,
+      "logps/rejected": -676.6248779296875,
+      "loss": 0.4414,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -2.2130393981933594,
+      "rewards/margins": 1.7717669010162354,
+      "rewards/rejected": -3.984806537628174,
+      "step": 4760
+    },
+    {
+      "epoch": 0.702193434417783,
+      "grad_norm": 130.0998503492656,
+      "learning_rate": 1.2337165128014443e-07,
+      "logits/chosen": -1.5202968120574951,
+      "logits/rejected": -1.3023194074630737,
+      "logps/chosen": -474.6209411621094,
+      "logps/rejected": -554.12255859375,
+      "loss": 0.5071,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.027803897857666,
+      "rewards/margins": 0.8622902631759644,
+      "rewards/rejected": -2.89009428024292,
+      "step": 4770
+    },
+    {
+      "epoch": 0.703665538053879,
+      "grad_norm": 250.14239845087963,
+      "learning_rate": 1.2226553268303494e-07,
+      "logits/chosen": -1.0853878259658813,
+      "logits/rejected": -0.8067598342895508,
+      "logps/chosen": -407.2760925292969,
+      "logps/rejected": -513.2825927734375,
+      "loss": 0.5553,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.9570262432098389,
+      "rewards/margins": 0.8497644662857056,
+      "rewards/rejected": -2.806790590286255,
+      "step": 4780
+    },
+    {
+      "epoch": 0.705137641689975,
+      "grad_norm": 99.66967115088167,
+      "learning_rate": 1.2116278772096835e-07,
+      "logits/chosen": -1.4983808994293213,
+      "logits/rejected": -1.0246336460113525,
+      "logps/chosen": -513.481689453125,
+      "logps/rejected": -591.3770751953125,
+      "loss": 0.4745,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -1.9119459390640259,
+      "rewards/margins": 1.4465692043304443,
+      "rewards/rejected": -3.3585147857666016,
+      "step": 4790
+    },
+    {
+      "epoch": 0.7066097453260709,
+      "grad_norm": 153.93818615763,
+      "learning_rate": 1.2006344551888736e-07,
+      "logits/chosen": -1.3309199810028076,
+      "logits/rejected": -1.3016533851623535,
+      "logps/chosen": -341.87969970703125,
+      "logps/rejected": -475.46685791015625,
+      "loss": 0.4456,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.5952895879745483,
+      "rewards/margins": 1.216280221939087,
+      "rewards/rejected": -2.811569929122925,
+      "step": 4800
+    },
+    {
+      "epoch": 0.708081848962167,
+      "grad_norm": 128.18739766292526,
+      "learning_rate": 1.1896753511186364e-07,
+      "logits/chosen": -1.6554391384124756,
+      "logits/rejected": -1.2733652591705322,
+      "logps/chosen": -459.8006896972656,
+      "logps/rejected": -439.5716857910156,
+      "loss": 0.4512,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -1.749202013015747,
+      "rewards/margins": 0.8417350053787231,
+      "rewards/rejected": -2.5909371376037598,
+      "step": 4810
+    },
+    {
+      "epoch": 0.7095539525982629,
+      "grad_norm": 113.03450199691848,
+      "learning_rate": 1.1787508544433002e-07,
+      "logits/chosen": -1.3578002452850342,
+      "logits/rejected": -1.318065881729126,
+      "logps/chosen": -434.5751037597656,
+      "logps/rejected": -565.3150024414062,
+      "loss": 0.5248,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.9271783828735352,
+      "rewards/margins": 1.20933997631073,
+      "rewards/rejected": -3.1365184783935547,
+      "step": 4820
+    },
+    {
+      "epoch": 0.7110260562343589,
+      "grad_norm": 71.73553787506627,
+      "learning_rate": 1.1678612536931718e-07,
+      "logits/chosen": -1.4692186117172241,
+      "logits/rejected": -1.2010374069213867,
+      "logps/chosen": -490.8218688964844,
+      "logps/rejected": -612.7962646484375,
+      "loss": 0.493,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.8638207912445068,
+      "rewards/margins": 1.580187439918518,
+      "rewards/rejected": -3.4440078735351562,
+      "step": 4830
+    },
+    {
+      "epoch": 0.7124981598704548,
+      "grad_norm": 161.8994182808025,
+      "learning_rate": 1.1570068364769081e-07,
+      "logits/chosen": -1.3370743989944458,
+      "logits/rejected": -1.150339961051941,
+      "logps/chosen": -488.45806884765625,
+      "logps/rejected": -614.4892578125,
+      "loss": 0.543,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -2.3346831798553467,
+      "rewards/margins": 0.9407814145088196,
+      "rewards/rejected": -3.2754645347595215,
+      "step": 4840
+    },
+    {
+      "epoch": 0.7139702635065509,
+      "grad_norm": 124.32759682385473,
+      "learning_rate": 1.146187889473924e-07,
+      "logits/chosen": -1.3396797180175781,
+      "logits/rejected": -1.2491532564163208,
+      "logps/chosen": -529.1656494140625,
+      "logps/rejected": -625.8582763671875,
+      "loss": 0.6059,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.514726400375366,
+      "rewards/margins": 1.1652178764343262,
+      "rewards/rejected": -3.679943799972534,
+      "step": 4850
+    },
+    {
+      "epoch": 0.7154423671426469,
+      "grad_norm": 106.6261258046526,
+      "learning_rate": 1.135404698426819e-07,
+      "logits/chosen": -1.412630319595337,
+      "logits/rejected": -1.2308682203292847,
+      "logps/chosen": -440.5975036621094,
+      "logps/rejected": -562.03125,
+      "loss": 0.4946,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.9255508184432983,
+      "rewards/margins": 1.1983132362365723,
+      "rewards/rejected": -3.123863697052002,
+      "step": 4860
+    },
+    {
+      "epoch": 0.7169144707787428,
+      "grad_norm": 216.01010311040795,
+      "learning_rate": 1.1246575481338305e-07,
+      "logits/chosen": -1.7150405645370483,
+      "logits/rejected": -1.3829970359802246,
+      "logps/chosen": -496.76922607421875,
+      "logps/rejected": -608.0321044921875,
+      "loss": 0.4743,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -1.8532264232635498,
+      "rewards/margins": 1.6127201318740845,
+      "rewards/rejected": -3.465946674346924,
+      "step": 4870
+    },
+    {
+      "epoch": 0.7183865744148388,
+      "grad_norm": 150.98279648882178,
+      "learning_rate": 1.1139467224413132e-07,
+      "logits/chosen": -1.4484059810638428,
+      "logits/rejected": -1.1759675741195679,
+      "logps/chosen": -431.6138610839844,
+      "logps/rejected": -513.0101928710938,
+      "loss": 0.5464,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9580078125,
+      "rewards/margins": 1.0588047504425049,
+      "rewards/rejected": -3.016812562942505,
+      "step": 4880
+    },
+    {
+      "epoch": 0.7198586780509347,
+      "grad_norm": 72.0822214592545,
+      "learning_rate": 1.1032725042362393e-07,
+      "logits/chosen": -1.4815934896469116,
+      "logits/rejected": -1.1622182130813599,
+      "logps/chosen": -507.9613342285156,
+      "logps/rejected": -515.327880859375,
+      "loss": 0.5333,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -2.3548049926757812,
+      "rewards/margins": 0.4555516839027405,
+      "rewards/rejected": -2.810356855392456,
+      "step": 4890
+    },
+    {
+      "epoch": 0.7213307816870308,
+      "grad_norm": 167.78987587220286,
+      "learning_rate": 1.0926351754387336e-07,
+      "logits/chosen": -1.4982646703720093,
+      "logits/rejected": -1.4483789205551147,
+      "logps/chosen": -490.54193115234375,
+      "logps/rejected": -559.3733520507812,
+      "loss": 0.4414,
+      "rewards/accuracies": 0.7333332300186157,
+      "rewards/chosen": -1.8408492803573608,
+      "rewards/margins": 1.072332739830017,
+      "rewards/rejected": -2.913182020187378,
+      "step": 4900
+    },
+    {
+      "epoch": 0.7228028853231268,
+      "grad_norm": 62.624564155928404,
+      "learning_rate": 1.0820350169946174e-07,
+      "logits/chosen": -1.639958143234253,
+      "logits/rejected": -1.617291808128357,
+      "logps/chosen": -365.8565979003906,
+      "logps/rejected": -504.843994140625,
+      "loss": 0.4516,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.3386435508728027,
+      "rewards/margins": 1.232550859451294,
+      "rewards/rejected": -2.5711944103240967,
+      "step": 4910
+    },
+    {
+      "epoch": 0.7242749889592227,
+      "grad_norm": 155.8678377271416,
+      "learning_rate": 1.0714723088679983e-07,
+      "logits/chosen": -1.6298201084136963,
+      "logits/rejected": -1.1968131065368652,
+      "logps/chosen": -499.95220947265625,
+      "logps/rejected": -512.9623413085938,
+      "loss": 0.4906,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.9635684490203857,
+      "rewards/margins": 1.0296690464019775,
+      "rewards/rejected": -2.993237018585205,
+      "step": 4920
+    },
+    {
+      "epoch": 0.7257470925953187,
+      "grad_norm": 49.24488724479236,
+      "learning_rate": 1.06094733003387e-07,
+      "logits/chosen": -1.343354344367981,
+      "logits/rejected": -1.3403180837631226,
+      "logps/chosen": -459.63055419921875,
+      "logps/rejected": -532.32568359375,
+      "loss": 0.4709,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.4064701795578003,
+      "rewards/margins": 1.1344773769378662,
+      "rewards/rejected": -2.540947675704956,
+      "step": 4930
+    },
+    {
+      "epoch": 0.7272191962314147,
+      "grad_norm": 188.84728835611227,
+      "learning_rate": 1.0504603584707463e-07,
+      "logits/chosen": -1.3937816619873047,
+      "logits/rejected": -1.1371171474456787,
+      "logps/chosen": -401.5303649902344,
+      "logps/rejected": -485.3896484375,
+      "loss": 0.5245,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.8510379791259766,
+      "rewards/margins": 0.8995128870010376,
+      "rewards/rejected": -2.7505507469177246,
+      "step": 4940
+    },
+    {
+      "epoch": 0.7286912998675107,
+      "grad_norm": 209.5458787823206,
+      "learning_rate": 1.0400116711533217e-07,
+      "logits/chosen": -1.495514988899231,
+      "logits/rejected": -1.0371195077896118,
+      "logps/chosen": -507.47802734375,
+      "logps/rejected": -536.724853515625,
+      "loss": 0.4967,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.639105200767517,
+      "rewards/margins": 0.920970618724823,
+      "rewards/rejected": -2.5600757598876953,
+      "step": 4950
+    },
+    {
+      "epoch": 0.7301634035036066,
+      "grad_norm": 195.11625946629115,
+      "learning_rate": 1.029601544045148e-07,
+      "logits/chosen": -1.2054619789123535,
+      "logits/rejected": -1.0448256731033325,
+      "logps/chosen": -476.5043029785156,
+      "logps/rejected": -516.1734619140625,
+      "loss": 0.4496,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.9478108882904053,
+      "rewards/margins": 0.9085144996643066,
+      "rewards/rejected": -2.856325149536133,
+      "step": 4960
+    },
+    {
+      "epoch": 0.7316355071397026,
+      "grad_norm": 94.34201740908914,
+      "learning_rate": 1.0192302520913563e-07,
+      "logits/chosen": -1.1716625690460205,
+      "logits/rejected": -0.9892371892929077,
+      "logps/chosen": -438.70098876953125,
+      "logps/rejected": -559.1282958984375,
+      "loss": 0.5305,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.879315972328186,
+      "rewards/margins": 1.1854360103607178,
+      "rewards/rejected": -3.0647521018981934,
+      "step": 4970
+    },
+    {
+      "epoch": 0.7331076107757987,
+      "grad_norm": 109.51344278681661,
+      "learning_rate": 1.0088980692113872e-07,
+      "logits/chosen": -1.1864292621612549,
+      "logits/rejected": -0.9743377566337585,
+      "logps/chosen": -546.6310424804688,
+      "logps/rejected": -661.9254150390625,
+      "loss": 0.4117,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -2.062819004058838,
+      "rewards/margins": 1.8060537576675415,
+      "rewards/rejected": -3.868872880935669,
+      "step": 4980
+    },
+    {
+      "epoch": 0.7345797144118946,
+      "grad_norm": 96.67830014928222,
+      "learning_rate": 9.986052682917611e-08,
+      "logits/chosen": -1.5318454504013062,
+      "logits/rejected": -1.413427710533142,
+      "logps/chosen": -524.6651611328125,
+      "logps/rejected": -549.9921264648438,
+      "loss": 0.6576,
+      "rewards/accuracies": 0.46666663885116577,
+      "rewards/chosen": -2.155850410461426,
+      "rewards/margins": 0.2569471001625061,
+      "rewards/rejected": -2.412797451019287,
+      "step": 4990
+    },
+    {
+      "epoch": 0.7360518180479906,
+      "grad_norm": 98.00148679324755,
+      "learning_rate": 9.883521211788682e-08,
+      "logits/chosen": -1.15413498878479,
+      "logits/rejected": -0.798266589641571,
+      "logps/chosen": -485.17510986328125,
+      "logps/rejected": -548.4317626953125,
+      "loss": 0.4739,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.8277232646942139,
+      "rewards/margins": 1.161808967590332,
+      "rewards/rejected": -2.989532470703125,
+      "step": 5000
+    },
+    {
+      "epoch": 0.7375239216840865,
+      "grad_norm": 71.61220605596463,
+      "learning_rate": 9.781388986717898e-08,
+      "logits/chosen": -1.2805821895599365,
+      "logits/rejected": -1.1228892803192139,
+      "logps/chosen": -395.42779541015625,
+      "logps/rejected": -517.1595458984375,
+      "loss": 0.4585,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.844408392906189,
+      "rewards/margins": 1.562734603881836,
+      "rewards/rejected": -3.4071431159973145,
+      "step": 5010
+    },
+    {
+      "epoch": 0.7389960253201825,
+      "grad_norm": 163.55873288904874,
+      "learning_rate": 9.679658705151461e-08,
+      "logits/chosen": -1.1837106943130493,
+      "logits/rejected": -0.9553543925285339,
+      "logps/chosen": -443.3148498535156,
+      "logps/rejected": -583.685791015625,
+      "loss": 0.5279,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.0693347454071045,
+      "rewards/margins": 1.5542361736297607,
+      "rewards/rejected": -3.6235709190368652,
+      "step": 5020
+    },
+    {
+      "epoch": 0.7404681289562786,
+      "grad_norm": 70.4876460256788,
+      "learning_rate": 9.578333053919704e-08,
+      "logits/chosen": -1.2908295392990112,
+      "logits/rejected": -1.153931736946106,
+      "logps/chosen": -418.4418029785156,
+      "logps/rejected": -645.3157348632812,
+      "loss": 0.5578,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.9433046579360962,
+      "rewards/margins": 1.7003980875015259,
+      "rewards/rejected": -3.643702745437622,
+      "step": 5030
+    },
+    {
+      "epoch": 0.7419402325923745,
+      "grad_norm": 67.46213633881179,
+      "learning_rate": 9.477414709166182e-08,
+      "logits/chosen": -1.42068350315094,
+      "logits/rejected": -1.0340807437896729,
+      "logps/chosen": -503.3772888183594,
+      "logps/rejected": -612.3836059570312,
+      "loss": 0.5481,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.9878009557724,
+      "rewards/margins": 1.3287580013275146,
+      "rewards/rejected": -3.316558837890625,
+      "step": 5040
+    },
+    {
+      "epoch": 0.7434123362284705,
+      "grad_norm": 72.50386150719055,
+      "learning_rate": 9.376906336276894e-08,
+      "logits/chosen": -1.139035940170288,
+      "logits/rejected": -0.9916818737983704,
+      "logps/chosen": -436.18255615234375,
+      "logps/rejected": -576.2352905273438,
+      "loss": 0.4961,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.7689425945281982,
+      "rewards/margins": 1.1512548923492432,
+      "rewards/rejected": -2.9201972484588623,
+      "step": 5050
+    },
+    {
+      "epoch": 0.7448844398645664,
+      "grad_norm": 97.50830907955964,
+      "learning_rate": 9.276810589809978e-08,
+      "logits/chosen": -1.1447765827178955,
+      "logits/rejected": -1.1658234596252441,
+      "logps/chosen": -352.49749755859375,
+      "logps/rejected": -486.12042236328125,
+      "loss": 0.496,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -1.6179523468017578,
+      "rewards/margins": 1.1843721866607666,
+      "rewards/rejected": -2.8023242950439453,
+      "step": 5060
+    },
+    {
+      "epoch": 0.7463565435006625,
+      "grad_norm": 333.1589456993549,
+      "learning_rate": 9.177130113425562e-08,
+      "logits/chosen": -1.1440602540969849,
+      "logits/rejected": -0.9470571279525757,
+      "logps/chosen": -411.8109436035156,
+      "logps/rejected": -523.9204711914062,
+      "loss": 0.5742,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.2491374015808105,
+      "rewards/margins": 1.1999796628952026,
+      "rewards/rejected": -3.4491169452667236,
+      "step": 5070
+    },
+    {
+      "epoch": 0.7478286471367585,
+      "grad_norm": 93.65396468667781,
+      "learning_rate": 9.077867539815948e-08,
+      "logits/chosen": -1.602186918258667,
+      "logits/rejected": -1.2702652215957642,
+      "logps/chosen": -421.89666748046875,
+      "logps/rejected": -470.13934326171875,
+      "loss": 0.4822,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.5477886199951172,
+      "rewards/margins": 1.0641921758651733,
+      "rewards/rejected": -2.61198091506958,
+      "step": 5080
+    },
+    {
+      "epoch": 0.7493007507728544,
+      "grad_norm": 34.029670357884406,
+      "learning_rate": 8.979025490636064e-08,
+      "logits/chosen": -1.3914169073104858,
+      "logits/rejected": -1.2724124193191528,
+      "logps/chosen": -453.03094482421875,
+      "logps/rejected": -518.9938354492188,
+      "loss": 0.4907,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.923095703125,
+      "rewards/margins": 0.8033218383789062,
+      "rewards/rejected": -2.7264175415039062,
+      "step": 5090
+    },
+    {
+      "epoch": 0.7507728544089504,
+      "grad_norm": 131.6843235646593,
+      "learning_rate": 8.880606576434249e-08,
+      "logits/chosen": -1.5269224643707275,
+      "logits/rejected": -1.27559232711792,
+      "logps/chosen": -420.00054931640625,
+      "logps/rejected": -526.8175659179688,
+      "loss": 0.476,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.71442449092865,
+      "rewards/margins": 1.1878302097320557,
+      "rewards/rejected": -2.902254819869995,
+      "step": 5100
+    },
+    {
+      "epoch": 0.7522449580450463,
+      "grad_norm": 63.167878671951094,
+      "learning_rate": 8.782613396583285e-08,
+      "logits/chosen": -1.3464393615722656,
+      "logits/rejected": -1.1066701412200928,
+      "logps/chosen": -465.05133056640625,
+      "logps/rejected": -484.81903076171875,
+      "loss": 0.5136,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.8139597177505493,
+      "rewards/margins": 0.6805374622344971,
+      "rewards/rejected": -2.494497537612915,
+      "step": 5110
+    },
+    {
+      "epoch": 0.7537170616811424,
+      "grad_norm": 64.74407065545697,
+      "learning_rate": 8.685048539211745e-08,
+      "logits/chosen": -1.1407724618911743,
+      "logits/rejected": -0.9420990943908691,
+      "logps/chosen": -507.9645080566406,
+      "logps/rejected": -594.517578125,
+      "loss": 0.4208,
+      "rewards/accuracies": 0.8000000715255737,
+      "rewards/chosen": -2.1703224182128906,
+      "rewards/margins": 1.167057752609253,
+      "rewards/rejected": -3.3373801708221436,
+      "step": 5120
+    },
+    {
+      "epoch": 0.7551891653172383,
+      "grad_norm": 96.23348450769105,
+      "learning_rate": 8.587914581135672e-08,
+      "logits/chosen": -1.0492364168167114,
+      "logits/rejected": -0.8715299367904663,
+      "logps/chosen": -418.671875,
+      "logps/rejected": -543.795654296875,
+      "loss": 0.4701,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.6521028280258179,
+      "rewards/margins": 1.2567085027694702,
+      "rewards/rejected": -2.908811330795288,
+      "step": 5130
+    },
+    {
+      "epoch": 0.7566612689533343,
+      "grad_norm": 241.03642964586655,
+      "learning_rate": 8.491214087790447e-08,
+      "logits/chosen": -1.5153532028198242,
+      "logits/rejected": -1.3691776990890503,
+      "logps/chosen": -522.0064697265625,
+      "logps/rejected": -659.2424926757812,
+      "loss": 0.5058,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.9308795928955078,
+      "rewards/margins": 1.337421178817749,
+      "rewards/rejected": -3.2683005332946777,
+      "step": 5140
+    },
+    {
+      "epoch": 0.7581333725894303,
+      "grad_norm": 117.21666272567893,
+      "learning_rate": 8.394949613163111e-08,
+      "logits/chosen": -1.1695173978805542,
+      "logits/rejected": -0.9150673151016235,
+      "logps/chosen": -395.46014404296875,
+      "logps/rejected": -482.9546813964844,
+      "loss": 0.4571,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.7090438604354858,
+      "rewards/margins": 1.1303540468215942,
+      "rewards/rejected": -2.83939790725708,
+      "step": 5150
+    },
+    {
+      "epoch": 0.7596054762255263,
+      "grad_norm": 49.60963349801694,
+      "learning_rate": 8.299123699724864e-08,
+      "logits/chosen": -1.254896879196167,
+      "logits/rejected": -0.8868924975395203,
+      "logps/chosen": -533.5479125976562,
+      "logps/rejected": -600.2605590820312,
+      "loss": 0.422,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -1.91446053981781,
+      "rewards/margins": 1.524327278137207,
+      "rewards/rejected": -3.4387879371643066,
+      "step": 5160
+    },
+    {
+      "epoch": 0.7610775798616223,
+      "grad_norm": 103.10433149190409,
+      "learning_rate": 8.203738878363933e-08,
+      "logits/chosen": -0.762785792350769,
+      "logits/rejected": -0.3832576870918274,
+      "logps/chosen": -445.01165771484375,
+      "logps/rejected": -505.7132873535156,
+      "loss": 0.5257,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.137202262878418,
+      "rewards/margins": 0.9919196963310242,
+      "rewards/rejected": -3.129122257232666,
+      "step": 5170
+    },
+    {
+      "epoch": 0.7625496834977182,
+      "grad_norm": 138.35907136125502,
+      "learning_rate": 8.108797668318743e-08,
+      "logits/chosen": -1.0560953617095947,
+      "logits/rejected": -0.9559084177017212,
+      "logps/chosen": -432.0171813964844,
+      "logps/rejected": -558.3382568359375,
+      "loss": 0.5275,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.146282434463501,
+      "rewards/margins": 1.1481616497039795,
+      "rewards/rejected": -3.2944445610046387,
+      "step": 5180
+    },
+    {
+      "epoch": 0.7640217871338142,
+      "grad_norm": 116.77510339573286,
+      "learning_rate": 8.01430257711132e-08,
+      "logits/chosen": -1.0193790197372437,
+      "logits/rejected": -0.8598148226737976,
+      "logps/chosen": -467.6903381347656,
+      "logps/rejected": -596.7313232421875,
+      "loss": 0.4548,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -2.212892532348633,
+      "rewards/margins": 1.633653998374939,
+      "rewards/rejected": -3.8465466499328613,
+      "step": 5190
+    },
+    {
+      "epoch": 0.7654938907699103,
+      "grad_norm": 229.39103812968878,
+      "learning_rate": 7.92025610048114e-08,
+      "logits/chosen": -1.185673475265503,
+      "logits/rejected": -0.8508531451225281,
+      "logps/chosen": -414.93475341796875,
+      "logps/rejected": -511.0469665527344,
+      "loss": 0.4651,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8168270587921143,
+      "rewards/margins": 1.3206034898757935,
+      "rewards/rejected": -3.13742995262146,
+      "step": 5200
+    },
+    {
+      "epoch": 0.7669659944060062,
+      "grad_norm": 145.52216912483013,
+      "learning_rate": 7.826660722319165e-08,
+      "logits/chosen": -1.303210735321045,
+      "logits/rejected": -0.8876203298568726,
+      "logps/chosen": -541.0342407226562,
+      "logps/rejected": -581.6346435546875,
+      "loss": 0.4307,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.7996604442596436,
+      "rewards/margins": 1.1814128160476685,
+      "rewards/rejected": -2.9810729026794434,
+      "step": 5210
+    },
+    {
+      "epoch": 0.7684380980421022,
+      "grad_norm": 108.17236548702299,
+      "learning_rate": 7.733518914602252e-08,
+      "logits/chosen": -1.5311328172683716,
+      "logits/rejected": -1.072148323059082,
+      "logps/chosen": -500.82989501953125,
+      "logps/rejected": -615.4925537109375,
+      "loss": 0.4553,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -1.6734565496444702,
+      "rewards/margins": 1.7444572448730469,
+      "rewards/rejected": -3.4179139137268066,
+      "step": 5220
+    },
+    {
+      "epoch": 0.7699102016781981,
+      "grad_norm": 70.65315062546658,
+      "learning_rate": 7.640833137327882e-08,
+      "logits/chosen": -1.0662109851837158,
+      "logits/rejected": -0.9030225872993469,
+      "logps/chosen": -433.59979248046875,
+      "logps/rejected": -546.4560546875,
+      "loss": 0.3907,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.6911451816558838,
+      "rewards/margins": 1.4692209959030151,
+      "rewards/rejected": -3.1603662967681885,
+      "step": 5230
+    },
+    {
+      "epoch": 0.7713823053142941,
+      "grad_norm": 180.98709386648449,
+      "learning_rate": 7.548605838449151e-08,
+      "logits/chosen": -0.8809256553649902,
+      "logits/rejected": -0.9900395274162292,
+      "logps/chosen": -410.05352783203125,
+      "logps/rejected": -558.8440551757812,
+      "loss": 0.4325,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.7245655059814453,
+      "rewards/margins": 1.144273042678833,
+      "rewards/rejected": -2.8688387870788574,
+      "step": 5240
+    },
+    {
+      "epoch": 0.7728544089503901,
+      "grad_norm": 124.88580544358221,
+      "learning_rate": 7.456839453810157e-08,
+      "logits/chosen": -1.2572662830352783,
+      "logits/rejected": -1.0659639835357666,
+      "logps/chosen": -494.0467834472656,
+      "logps/rejected": -647.9281005859375,
+      "loss": 0.4469,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.1124119758605957,
+      "rewards/margins": 1.4500975608825684,
+      "rewards/rejected": -3.5625100135803223,
+      "step": 5250
+    },
+    {
+      "epoch": 0.7743265125864861,
+      "grad_norm": 154.45317247906624,
+      "learning_rate": 7.365536407081633e-08,
+      "logits/chosen": -1.251163363456726,
+      "logits/rejected": -1.229315161705017,
+      "logps/chosen": -428.061279296875,
+      "logps/rejected": -601.31689453125,
+      "loss": 0.404,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.6897375583648682,
+      "rewards/margins": 1.5642296075820923,
+      "rewards/rejected": -3.253966808319092,
+      "step": 5260
+    },
+    {
+      "epoch": 0.7757986162225821,
+      "grad_norm": 85.77593336032358,
+      "learning_rate": 7.274699109696975e-08,
+      "logits/chosen": -1.0087244510650635,
+      "logits/rejected": -0.6363364458084106,
+      "logps/chosen": -390.22314453125,
+      "logps/rejected": -438.15655517578125,
+      "loss": 0.549,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.5335338115692139,
+      "rewards/margins": 0.8562105298042297,
+      "rewards/rejected": -2.389744520187378,
+      "step": 5270
+    },
+    {
+      "epoch": 0.777270719858678,
+      "grad_norm": 155.2419340848915,
+      "learning_rate": 7.184329960788491e-08,
+      "logits/chosen": -1.0169925689697266,
+      "logits/rejected": -0.7420837879180908,
+      "logps/chosen": -395.13763427734375,
+      "logps/rejected": -565.1148681640625,
+      "loss": 0.4765,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8920724391937256,
+      "rewards/margins": 1.4287753105163574,
+      "rewards/rejected": -3.320847988128662,
+      "step": 5280
+    },
+    {
+      "epoch": 0.7787428234947741,
+      "grad_norm": 87.81525453347075,
+      "learning_rate": 7.094431347124092e-08,
+      "logits/chosen": -0.8687102198600769,
+      "logits/rejected": -0.6938323974609375,
+      "logps/chosen": -404.7712707519531,
+      "logps/rejected": -538.3594970703125,
+      "loss": 0.3722,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.5631678104400635,
+      "rewards/margins": 1.6385186910629272,
+      "rewards/rejected": -3.201686382293701,
+      "step": 5290
+    },
+    {
+      "epoch": 0.78021492713087,
+      "grad_norm": 59.173002113288824,
+      "learning_rate": 7.00500564304424e-08,
+      "logits/chosen": -1.2083094120025635,
+      "logits/rejected": -0.9714385271072388,
+      "logps/chosen": -557.9366455078125,
+      "logps/rejected": -621.3553466796875,
+      "loss": 0.4425,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.9470123052597046,
+      "rewards/margins": 1.1327309608459473,
+      "rewards/rejected": -3.0797431468963623,
+      "step": 5300
+    },
+    {
+      "epoch": 0.781687030766966,
+      "grad_norm": 215.11128920992735,
+      "learning_rate": 6.916055210399219e-08,
+      "logits/chosen": -1.3841431140899658,
+      "logits/rejected": -1.386099100112915,
+      "logps/chosen": -473.8697814941406,
+      "logps/rejected": -637.9520263671875,
+      "loss": 0.539,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7102562189102173,
+      "rewards/margins": 1.2155544757843018,
+      "rewards/rejected": -2.9258105754852295,
+      "step": 5310
+    },
+    {
+      "epoch": 0.783159134403062,
+      "grad_norm": 225.67677440514703,
+      "learning_rate": 6.827582398486797e-08,
+      "logits/chosen": -1.2445056438446045,
+      "logits/rejected": -0.9456964731216431,
+      "logps/chosen": -578.6286010742188,
+      "logps/rejected": -627.7783203125,
+      "loss": 0.6115,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -1.9115711450576782,
+      "rewards/margins": 1.503936529159546,
+      "rewards/rejected": -3.4155075550079346,
+      "step": 5320
+    },
+    {
+      "epoch": 0.7846312380391579,
+      "grad_norm": 77.17729472372167,
+      "learning_rate": 6.739589543990118e-08,
+      "logits/chosen": -0.9577809572219849,
+      "logits/rejected": -0.5505359768867493,
+      "logps/chosen": -462.0064392089844,
+      "logps/rejected": -527.6776123046875,
+      "loss": 0.5602,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.7571370601654053,
+      "rewards/margins": 1.3212145566940308,
+      "rewards/rejected": -3.0783514976501465,
+      "step": 5330
+    },
+    {
+      "epoch": 0.786103341675254,
+      "grad_norm": 109.22908357628783,
+      "learning_rate": 6.652078970916037e-08,
+      "logits/chosen": -1.3481218814849854,
+      "logits/rejected": -1.0555284023284912,
+      "logps/chosen": -458.81292724609375,
+      "logps/rejected": -632.86572265625,
+      "loss": 0.45,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.9127309322357178,
+      "rewards/margins": 1.7171541452407837,
+      "rewards/rejected": -3.629885196685791,
+      "step": 5340
+    },
+    {
+      "epoch": 0.7875754453113499,
+      "grad_norm": 53.024674570795995,
+      "learning_rate": 6.565052990533715e-08,
+      "logits/chosen": -1.2405158281326294,
+      "logits/rejected": -1.1840492486953735,
+      "logps/chosen": -474.354248046875,
+      "logps/rejected": -581.2073364257812,
+      "loss": 0.494,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.223118305206299,
+      "rewards/margins": 1.165349006652832,
+      "rewards/rejected": -3.388467311859131,
+      "step": 5350
+    },
+    {
+      "epoch": 0.7890475489474459,
+      "grad_norm": 238.66915414797674,
+      "learning_rate": 6.478513901313615e-08,
+      "logits/chosen": -0.9796088933944702,
+      "logits/rejected": -0.9768195152282715,
+      "logps/chosen": -406.1162109375,
+      "logps/rejected": -567.86328125,
+      "loss": 0.5971,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.611419677734375,
+      "rewards/margins": 1.1555076837539673,
+      "rewards/rejected": -2.766927480697632,
+      "step": 5360
+    },
+    {
+      "epoch": 0.7905196525835418,
+      "grad_norm": 140.70071817008576,
+      "learning_rate": 6.392463988866714e-08,
+      "logits/chosen": -1.0570393800735474,
+      "logits/rejected": -0.9775602221488953,
+      "logps/chosen": -456.1234436035156,
+      "logps/rejected": -606.5938110351562,
+      "loss": 0.5794,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.8095552921295166,
+      "rewards/margins": 1.2307498455047607,
+      "rewards/rejected": -3.0403048992156982,
+      "step": 5370
+    },
+    {
+      "epoch": 0.7919917562196379,
+      "grad_norm": 52.10965788819851,
+      "learning_rate": 6.306905525884227e-08,
+      "logits/chosen": -1.5277817249298096,
+      "logits/rejected": -1.355376958847046,
+      "logps/chosen": -475.4259338378906,
+      "logps/rejected": -586.855224609375,
+      "loss": 0.4918,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0407283306121826,
+      "rewards/margins": 1.0685203075408936,
+      "rewards/rejected": -3.109248638153076,
+      "step": 5380
+    },
+    {
+      "epoch": 0.7934638598557339,
+      "grad_norm": 175.36706272305227,
+      "learning_rate": 6.221840772077525e-08,
+      "logits/chosen": -1.2016507387161255,
+      "logits/rejected": -1.0885107517242432,
+      "logps/chosen": -439.05462646484375,
+      "logps/rejected": -516.28515625,
+      "loss": 0.4896,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -2.0163323879241943,
+      "rewards/margins": 1.0620869398117065,
+      "rewards/rejected": -3.0784192085266113,
+      "step": 5390
+    },
+    {
+      "epoch": 0.7949359634918298,
+      "grad_norm": 105.3003864094294,
+      "learning_rate": 6.137271974118468e-08,
+      "logits/chosen": -1.4145647287368774,
+      "logits/rejected": -1.1715717315673828,
+      "logps/chosen": -466.78466796875,
+      "logps/rejected": -536.9066162109375,
+      "loss": 0.4988,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -2.034123659133911,
+      "rewards/margins": 0.9236257672309875,
+      "rewards/rejected": -2.957749366760254,
+      "step": 5400
+    },
+    {
+      "epoch": 0.7964080671279258,
+      "grad_norm": 76.8887580233539,
+      "learning_rate": 6.05320136558011e-08,
+      "logits/chosen": -1.4233685731887817,
+      "logits/rejected": -1.3150105476379395,
+      "logps/chosen": -439.65411376953125,
+      "logps/rejected": -577.3585815429688,
+      "loss": 0.5535,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": -1.8920074701309204,
+      "rewards/margins": 1.148402452468872,
+      "rewards/rejected": -3.040409564971924,
+      "step": 5410
+    },
+    {
+      "epoch": 0.7978801707640217,
+      "grad_norm": 123.14342704766716,
+      "learning_rate": 5.969631166877607e-08,
+      "logits/chosen": -1.337247610092163,
+      "logits/rejected": -0.9550000429153442,
+      "logps/chosen": -489.42742919921875,
+      "logps/rejected": -566.384765625,
+      "loss": 0.4342,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.637761116027832,
+      "rewards/margins": 1.4270638227462769,
+      "rewards/rejected": -3.0648245811462402,
+      "step": 5420
+    },
+    {
+      "epoch": 0.7993522744001178,
+      "grad_norm": 115.54034393158733,
+      "learning_rate": 5.8865635852096754e-08,
+      "logits/chosen": -1.2357947826385498,
+      "logits/rejected": -1.0595812797546387,
+      "logps/chosen": -422.2577209472656,
+      "logps/rejected": -501.9385681152344,
+      "loss": 0.4183,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.8146345615386963,
+      "rewards/margins": 1.0913867950439453,
+      "rewards/rejected": -2.9060215950012207,
+      "step": 5430
+    },
+    {
+      "epoch": 0.8008243780362138,
+      "grad_norm": 157.6274293347524,
+      "learning_rate": 5.8040008145002344e-08,
+      "logits/chosen": -1.7033611536026,
+      "logits/rejected": -0.9675869941711426,
+      "logps/chosen": -513.8781127929688,
+      "logps/rejected": -606.8839111328125,
+      "loss": 0.414,
+      "rewards/accuracies": 0.9000000953674316,
+      "rewards/chosen": -1.8686834573745728,
+      "rewards/margins": 1.5403255224227905,
+      "rewards/rejected": -3.409008741378784,
+      "step": 5440
+    },
+    {
+      "epoch": 0.8022964816723097,
+      "grad_norm": 111.81350505285386,
+      "learning_rate": 5.721945035340511e-08,
+      "logits/chosen": -1.1051543951034546,
+      "logits/rejected": -0.8956894874572754,
+      "logps/chosen": -411.0330505371094,
+      "logps/rejected": -571.52880859375,
+      "loss": 0.429,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.614373803138733,
+      "rewards/margins": 1.9639537334442139,
+      "rewards/rejected": -3.5783278942108154,
+      "step": 5450
+    },
+    {
+      "epoch": 0.8037685853084057,
+      "grad_norm": 101.85766958151592,
+      "learning_rate": 5.640398414931399e-08,
+      "logits/chosen": -1.013607382774353,
+      "logits/rejected": -1.0927541255950928,
+      "logps/chosen": -410.86724853515625,
+      "logps/rejected": -541.094482421875,
+      "loss": 0.633,
+      "rewards/accuracies": 0.5999999642372131,
+      "rewards/chosen": -2.005836009979248,
+      "rewards/margins": 0.868213951587677,
+      "rewards/rejected": -2.874049663543701,
+      "step": 5460
+    },
+    {
+      "epoch": 0.8052406889445017,
+      "grad_norm": 163.6961418271051,
+      "learning_rate": 5.5593631070262296e-08,
+      "logits/chosen": -1.2479948997497559,
+      "logits/rejected": -0.8561438322067261,
+      "logps/chosen": -516.5875244140625,
+      "logps/rejected": -675.2960815429688,
+      "loss": 0.4573,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -2.2040138244628906,
+      "rewards/margins": 1.7423547506332397,
+      "rewards/rejected": -3.946368455886841,
+      "step": 5470
+    },
+    {
+      "epoch": 0.8067127925805977,
+      "grad_norm": 74.07186173313391,
+      "learning_rate": 5.478841251873922e-08,
+      "logits/chosen": -1.3214751482009888,
+      "logits/rejected": -0.9202545285224915,
+      "logps/chosen": -486.7603454589844,
+      "logps/rejected": -577.1681518554688,
+      "loss": 0.4266,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.4990867376327515,
+      "rewards/margins": 1.670100212097168,
+      "rewards/rejected": -3.169187068939209,
+      "step": 5480
+    },
+    {
+      "epoch": 0.8081848962166936,
+      "grad_norm": 59.471481247489244,
+      "learning_rate": 5.398834976162414e-08,
+      "logits/chosen": -1.2711598873138428,
+      "logits/rejected": -1.0352222919464111,
+      "logps/chosen": -439.3692932128906,
+      "logps/rejected": -608.0352172851562,
+      "loss": 0.4223,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.5429414510726929,
+      "rewards/margins": 1.9663690328598022,
+      "rewards/rejected": -3.509310245513916,
+      "step": 5490
+    },
+    {
+      "epoch": 0.8096569998527896,
+      "grad_norm": 92.59368398233123,
+      "learning_rate": 5.319346392962545e-08,
+      "logits/chosen": -1.1628122329711914,
+      "logits/rejected": -1.3315460681915283,
+      "logps/chosen": -462.98309326171875,
+      "logps/rejected": -596.800048828125,
+      "loss": 0.4933,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.9284751415252686,
+      "rewards/margins": 1.2932318449020386,
+      "rewards/rejected": -3.2217068672180176,
+      "step": 5500
+    },
+    {
+      "epoch": 0.8111291034888857,
+      "grad_norm": 115.56862692289259,
+      "learning_rate": 5.24037760167218e-08,
+      "logits/chosen": -1.342844843864441,
+      "logits/rejected": -1.0954813957214355,
+      "logps/chosen": -491.49053955078125,
+      "logps/rejected": -619.1533813476562,
+      "loss": 0.4144,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.6697956323623657,
+      "rewards/margins": 1.7845573425292969,
+      "rewards/rejected": -3.454352855682373,
+      "step": 5510
+    },
+    {
+      "epoch": 0.8126012071249816,
+      "grad_norm": 125.46726804948204,
+      "learning_rate": 5.161930687960808e-08,
+      "logits/chosen": -1.2879449129104614,
+      "logits/rejected": -0.6370636224746704,
+      "logps/chosen": -508.4657287597656,
+      "logps/rejected": -519.69921875,
+      "loss": 0.5967,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -2.0370121002197266,
+      "rewards/margins": 1.1051788330078125,
+      "rewards/rejected": -3.142190933227539,
+      "step": 5520
+    },
+    {
+      "epoch": 0.8140733107610776,
+      "grad_norm": 179.96357276317437,
+      "learning_rate": 5.0840077237144594e-08,
+      "logits/chosen": -1.3536159992218018,
+      "logits/rejected": -0.7188901901245117,
+      "logps/chosen": -476.391357421875,
+      "logps/rejected": -564.7723999023438,
+      "loss": 0.4715,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.8136851787567139,
+      "rewards/margins": 1.4985334873199463,
+      "rewards/rejected": -3.312218427658081,
+      "step": 5530
+    },
+    {
+      "epoch": 0.8155454143971735,
+      "grad_norm": 137.98162018971811,
+      "learning_rate": 5.006610766980945e-08,
+      "logits/chosen": -1.4704669713974,
+      "logits/rejected": -1.1928757429122925,
+      "logps/chosen": -478.86285400390625,
+      "logps/rejected": -580.6500854492188,
+      "loss": 0.5194,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.861915946006775,
+      "rewards/margins": 1.3256458044052124,
+      "rewards/rejected": -3.1875619888305664,
+      "step": 5540
+    },
+    {
+      "epoch": 0.8170175180332695,
+      "grad_norm": 183.42102471753302,
+      "learning_rate": 4.929741861915571e-08,
+      "logits/chosen": -1.4357191324234009,
+      "logits/rejected": -1.1636178493499756,
+      "logps/chosen": -431.68670654296875,
+      "logps/rejected": -512.5819091796875,
+      "loss": 0.4701,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.772092580795288,
+      "rewards/margins": 1.139080286026001,
+      "rewards/rejected": -2.911172866821289,
+      "step": 5550
+    },
+    {
+      "epoch": 0.8184896216693656,
+      "grad_norm": 198.6027603437246,
+      "learning_rate": 4.853403038727047e-08,
+      "logits/chosen": -1.0890223979949951,
+      "logits/rejected": -1.047532081604004,
+      "logps/chosen": -422.9931640625,
+      "logps/rejected": -612.4053344726562,
+      "loss": 0.5113,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.9905509948730469,
+      "rewards/margins": 1.0186119079589844,
+      "rewards/rejected": -3.009162425994873,
+      "step": 5560
+    },
+    {
+      "epoch": 0.8199617253054615,
+      "grad_norm": 168.23022056516947,
+      "learning_rate": 4.777596313623966e-08,
+      "logits/chosen": -1.2151178121566772,
+      "logits/rejected": -1.0620383024215698,
+      "logps/chosen": -509.9613342285156,
+      "logps/rejected": -609.4564208984375,
+      "loss": 0.4881,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.6974384784698486,
+      "rewards/margins": 1.595670461654663,
+      "rewards/rejected": -3.293109178543091,
+      "step": 5570
+    },
+    {
+      "epoch": 0.8214338289415575,
+      "grad_norm": 72.3016109109807,
+      "learning_rate": 4.702323688761492e-08,
+      "logits/chosen": -1.3811545372009277,
+      "logits/rejected": -0.8901753425598145,
+      "logps/chosen": -473.8961486816406,
+      "logps/rejected": -604.1268310546875,
+      "loss": 0.4628,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -2.2072179317474365,
+      "rewards/margins": 1.3967196941375732,
+      "rewards/rejected": -3.603937864303589,
+      "step": 5580
+    },
+    {
+      "epoch": 0.8229059325776534,
+      "grad_norm": 135.1282785356735,
+      "learning_rate": 4.627587152188522e-08,
+      "logits/chosen": -1.446920394897461,
+      "logits/rejected": -1.1027758121490479,
+      "logps/chosen": -443.819091796875,
+      "logps/rejected": -525.2467041015625,
+      "loss": 0.4622,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.6185214519500732,
+      "rewards/margins": 1.4930517673492432,
+      "rewards/rejected": -3.1115729808807373,
+      "step": 5590
+    },
+    {
+      "epoch": 0.8243780362137495,
+      "grad_norm": 52.554048763874356,
+      "learning_rate": 4.5533886777951395e-08,
+      "logits/chosen": -1.4271553754806519,
+      "logits/rejected": -1.3268260955810547,
+      "logps/chosen": -479.57855224609375,
+      "logps/rejected": -691.8262939453125,
+      "loss": 0.4729,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.5048121213912964,
+      "rewards/margins": 1.455271601676941,
+      "rewards/rejected": -2.960083484649658,
+      "step": 5600
+    },
+    {
+      "epoch": 0.8258501398498455,
+      "grad_norm": 177.29394626551294,
+      "learning_rate": 4.4797302252604916e-08,
+      "logits/chosen": -1.188375473022461,
+      "logits/rejected": -0.9038418531417847,
+      "logps/chosen": -409.9644470214844,
+      "logps/rejected": -406.25323486328125,
+      "loss": 0.5234,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.814875841140747,
+      "rewards/margins": 0.5175659656524658,
+      "rewards/rejected": -2.332441806793213,
+      "step": 5610
+    },
+    {
+      "epoch": 0.8273222434859414,
+      "grad_norm": 119.22624538847619,
+      "learning_rate": 4.4066137400010555e-08,
+      "logits/chosen": -1.355344295501709,
+      "logits/rejected": -0.9832057952880859,
+      "logps/chosen": -445.841796875,
+      "logps/rejected": -641.5125732421875,
+      "loss": 0.4839,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.7873265743255615,
+      "rewards/margins": 2.185568332672119,
+      "rewards/rejected": -3.9728951454162598,
+      "step": 5620
+    },
+    {
+      "epoch": 0.8287943471220374,
+      "grad_norm": 137.0047831227428,
+      "learning_rate": 4.3340411531192306e-08,
+      "logits/chosen": -1.5107392072677612,
+      "logits/rejected": -1.3040306568145752,
+      "logps/chosen": -445.95208740234375,
+      "logps/rejected": -476.7379455566406,
+      "loss": 0.4406,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.3861254453659058,
+      "rewards/margins": 1.0516457557678223,
+      "rewards/rejected": -2.4377713203430176,
+      "step": 5630
+    },
+    {
+      "epoch": 0.8302664507581333,
+      "grad_norm": 82.84525438977428,
+      "learning_rate": 4.262014381352363e-08,
+      "logits/chosen": -1.3577890396118164,
+      "logits/rejected": -0.9183663129806519,
+      "logps/chosen": -452.920654296875,
+      "logps/rejected": -557.4637451171875,
+      "loss": 0.5162,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.7773888111114502,
+      "rewards/margins": 1.2821528911590576,
+      "rewards/rejected": -3.0595414638519287,
+      "step": 5640
+    },
+    {
+      "epoch": 0.8317385543942294,
+      "grad_norm": 76.17226865199585,
+      "learning_rate": 4.1905353270220795e-08,
+      "logits/chosen": -1.280076265335083,
+      "logits/rejected": -0.932104229927063,
+      "logps/chosen": -393.5255126953125,
+      "logps/rejected": -562.3932495117188,
+      "loss": 0.5413,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.365314245223999,
+      "rewards/margins": 1.344012975692749,
+      "rewards/rejected": -2.709327459335327,
+      "step": 5650
+    },
+    {
+      "epoch": 0.8332106580303253,
+      "grad_norm": 121.3359110614011,
+      "learning_rate": 4.119605877984089e-08,
+      "logits/chosen": -1.3621351718902588,
+      "logits/rejected": -1.2421245574951172,
+      "logps/chosen": -403.90557861328125,
+      "logps/rejected": -476.821533203125,
+      "loss": 0.7086,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.9038448333740234,
+      "rewards/margins": 0.5362643003463745,
+      "rewards/rejected": -2.4401090145111084,
+      "step": 5660
+    },
+    {
+      "epoch": 0.8346827616664213,
+      "grad_norm": 83.12577355392818,
+      "learning_rate": 4.049227907578284e-08,
+      "logits/chosen": -1.2348663806915283,
+      "logits/rejected": -1.2635643482208252,
+      "logps/chosen": -422.54376220703125,
+      "logps/rejected": -676.0519409179688,
+      "loss": 0.4528,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.659354567527771,
+      "rewards/margins": 2.0648245811462402,
+      "rewards/rejected": -3.7241787910461426,
+      "step": 5670
+    },
+    {
+      "epoch": 0.8361548653025173,
+      "grad_norm": 116.01745966168035,
+      "learning_rate": 3.979403274579313e-08,
+      "logits/chosen": -1.3450870513916016,
+      "logits/rejected": -0.9937416315078735,
+      "logps/chosen": -479.0702209472656,
+      "logps/rejected": -600.4904174804688,
+      "loss": 0.4331,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.8521366119384766,
+      "rewards/margins": 1.6752088069915771,
+      "rewards/rejected": -3.527345657348633,
+      "step": 5680
+    },
+    {
+      "epoch": 0.8376269689386133,
+      "grad_norm": 211.47659985662133,
+      "learning_rate": 3.9101338231474354e-08,
+      "logits/chosen": -1.4885746240615845,
+      "logits/rejected": -1.256639838218689,
+      "logps/chosen": -473.7394104003906,
+      "logps/rejected": -589.0265502929688,
+      "loss": 0.5276,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.9350303411483765,
+      "rewards/margins": 1.2548571825027466,
+      "rewards/rejected": -3.189887523651123,
+      "step": 5690
+    },
+    {
+      "epoch": 0.8390990725747093,
+      "grad_norm": 205.77550522367355,
+      "learning_rate": 3.841421382779827e-08,
+      "logits/chosen": -1.3588941097259521,
+      "logits/rejected": -1.2172741889953613,
+      "logps/chosen": -419.08978271484375,
+      "logps/rejected": -616.7435302734375,
+      "loss": 0.4747,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.7490804195404053,
+      "rewards/margins": 1.648638367652893,
+      "rewards/rejected": -3.397718906402588,
+      "step": 5700
+    },
+    {
+      "epoch": 0.8405711762108052,
+      "grad_norm": 102.47784262945792,
+      "learning_rate": 3.773267768262289e-08,
+      "logits/chosen": -1.27297842502594,
+      "logits/rejected": -0.9850039482116699,
+      "logps/chosen": -427.94305419921875,
+      "logps/rejected": -562.6896362304688,
+      "loss": 0.5676,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.608070731163025,
+      "rewards/margins": 1.3251826763153076,
+      "rewards/rejected": -2.933253526687622,
+      "step": 5710
+    },
+    {
+      "epoch": 0.8420432798469012,
+      "grad_norm": 71.0823358327679,
+      "learning_rate": 3.7056747796212844e-08,
+      "logits/chosen": -1.2771185636520386,
+      "logits/rejected": -0.8199060559272766,
+      "logps/chosen": -396.62384033203125,
+      "logps/rejected": -527.8873901367188,
+      "loss": 0.4536,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.4784718751907349,
+      "rewards/margins": 1.6878697872161865,
+      "rewards/rejected": -3.166341543197632,
+      "step": 5720
+    },
+    {
+      "epoch": 0.8435153834829971,
+      "grad_norm": 101.30985808697004,
+      "learning_rate": 3.63864420207643e-08,
+      "logits/chosen": -1.4370369911193848,
+      "logits/rejected": -1.2566996812820435,
+      "logps/chosen": -465.5155334472656,
+      "logps/rejected": -630.5440673828125,
+      "loss": 0.3811,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -1.6955715417861938,
+      "rewards/margins": 1.8823350667953491,
+      "rewards/rejected": -3.577906847000122,
+      "step": 5730
+    },
+    {
+      "epoch": 0.8449874871190932,
+      "grad_norm": 100.84237966720134,
+      "learning_rate": 3.5721778059933096e-08,
+      "logits/chosen": -1.3979724645614624,
+      "logits/rejected": -1.179774522781372,
+      "logps/chosen": -445.75372314453125,
+      "logps/rejected": -442.28839111328125,
+      "loss": 0.5404,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.8010488748550415,
+      "rewards/margins": 0.5255202651023865,
+      "rewards/rejected": -2.3265693187713623,
+      "step": 5740
+    },
+    {
+      "epoch": 0.8464595907551892,
+      "grad_norm": 60.767936064669804,
+      "learning_rate": 3.506277346836731e-08,
+      "logits/chosen": -1.1453638076782227,
+      "logits/rejected": -1.4349292516708374,
+      "logps/chosen": -452.9850158691406,
+      "logps/rejected": -645.1492919921875,
+      "loss": 0.5655,
+      "rewards/accuracies": 0.5333333015441895,
+      "rewards/chosen": -2.2502622604370117,
+      "rewards/margins": 0.9999488592147827,
+      "rewards/rejected": -3.250211238861084,
+      "step": 5750
+    },
+    {
+      "epoch": 0.8479316943912851,
+      "grad_norm": 152.6111398448886,
+      "learning_rate": 3.440944565124371e-08,
+      "logits/chosen": -1.000203013420105,
+      "logits/rejected": -0.7757139801979065,
+      "logps/chosen": -362.6944885253906,
+      "logps/rejected": -487.8475646972656,
+      "loss": 0.3965,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.3980891704559326,
+      "rewards/margins": 1.1738362312316895,
+      "rewards/rejected": -2.571925640106201,
+      "step": 5760
+    },
+    {
+      "epoch": 0.8494037980273811,
+      "grad_norm": 163.80390518535975,
+      "learning_rate": 3.376181186380808e-08,
+      "logits/chosen": -1.407921552658081,
+      "logits/rejected": -1.1443045139312744,
+      "logps/chosen": -446.7001953125,
+      "logps/rejected": -488.558349609375,
+      "loss": 0.5739,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -2.02390718460083,
+      "rewards/margins": 0.6599318385124207,
+      "rewards/rejected": -2.6838388442993164,
+      "step": 5770
+    },
+    {
+      "epoch": 0.8508759016634772,
+      "grad_norm": 213.6345979004298,
+      "learning_rate": 3.311988921091935e-08,
+      "logits/chosen": -1.5590388774871826,
+      "logits/rejected": -1.0573437213897705,
+      "logps/chosen": -445.07977294921875,
+      "logps/rejected": -486.111083984375,
+      "loss": 0.4314,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.4946223497390747,
+      "rewards/margins": 1.2161999940872192,
+      "rewards/rejected": -2.710822343826294,
+      "step": 5780
+    },
+    {
+      "epoch": 0.8523480052995731,
+      "grad_norm": 114.36351797505455,
+      "learning_rate": 3.248369464659775e-08,
+      "logits/chosen": -1.1733253002166748,
+      "logits/rejected": -1.0773212909698486,
+      "logps/chosen": -366.6900634765625,
+      "logps/rejected": -502.72845458984375,
+      "loss": 0.4712,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.5471848249435425,
+      "rewards/margins": 1.5482925176620483,
+      "rewards/rejected": -3.0954771041870117,
+      "step": 5790
+    },
+    {
+      "epoch": 0.8538201089356691,
+      "grad_norm": 86.24945234823068,
+      "learning_rate": 3.1853244973577306e-08,
+      "logits/chosen": -1.417776107788086,
+      "logits/rejected": -0.8327361941337585,
+      "logps/chosen": -466.37066650390625,
+      "logps/rejected": -610.36865234375,
+      "loss": 0.5163,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.8226988315582275,
+      "rewards/margins": 1.4830236434936523,
+      "rewards/rejected": -3.30572247505188,
+      "step": 5800
+    },
+    {
+      "epoch": 0.855292212571765,
+      "grad_norm": 82.26745113001945,
+      "learning_rate": 3.122855684286185e-08,
+      "logits/chosen": -1.0725051164627075,
+      "logits/rejected": -0.935965895652771,
+      "logps/chosen": -491.5113220214844,
+      "logps/rejected": -638.8419189453125,
+      "loss": 0.4563,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.990782380104065,
+      "rewards/margins": 1.335800290107727,
+      "rewards/rejected": -3.326582670211792,
+      "step": 5810
+    },
+    {
+      "epoch": 0.8567643162078611,
+      "grad_norm": 62.483405589834824,
+      "learning_rate": 3.060964675328545e-08,
+      "logits/chosen": -1.1046890020370483,
+      "logits/rejected": -0.9802217483520508,
+      "logps/chosen": -476.57305908203125,
+      "logps/rejected": -584.443359375,
+      "loss": 0.5224,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.23058819770813,
+      "rewards/margins": 0.9979446530342102,
+      "rewards/rejected": -3.2285327911376953,
+      "step": 5820
+    },
+    {
+      "epoch": 0.858236419843957,
+      "grad_norm": 130.02543143459314,
+      "learning_rate": 2.999653105107644e-08,
+      "logits/chosen": -1.2632063627243042,
+      "logits/rejected": -1.0915645360946655,
+      "logps/chosen": -381.5784912109375,
+      "logps/rejected": -556.1746826171875,
+      "loss": 0.5175,
+      "rewards/accuracies": 0.9000000953674316,
+      "rewards/chosen": -1.3862593173980713,
+      "rewards/margins": 1.8085914850234985,
+      "rewards/rejected": -3.1948509216308594,
+      "step": 5830
+    },
+    {
+      "epoch": 0.859708523480053,
+      "grad_norm": 97.1206867972697,
+      "learning_rate": 2.9389225929425637e-08,
+      "logits/chosen": -1.3377158641815186,
+      "logits/rejected": -1.2011384963989258,
+      "logps/chosen": -396.65777587890625,
+      "logps/rejected": -504.1952209472656,
+      "loss": 0.4344,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.5112955570220947,
+      "rewards/margins": 1.373363971710205,
+      "rewards/rejected": -2.8846595287323,
+      "step": 5840
+    },
+    {
+      "epoch": 0.861180627116149,
+      "grad_norm": 169.86234185913474,
+      "learning_rate": 2.8787747428058977e-08,
+      "logits/chosen": -1.3013274669647217,
+      "logits/rejected": -1.3599807024002075,
+      "logps/chosen": -475.5846252441406,
+      "logps/rejected": -583.6883544921875,
+      "loss": 0.4146,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.5154919624328613,
+      "rewards/margins": 1.2250664234161377,
+      "rewards/rejected": -2.740558385848999,
+      "step": 5850
+    },
+    {
+      "epoch": 0.8626527307522449,
+      "grad_norm": 93.00186430580612,
+      "learning_rate": 2.8192111432813554e-08,
+      "logits/chosen": -1.704040765762329,
+      "logits/rejected": -1.2342931032180786,
+      "logps/chosen": -447.02362060546875,
+      "logps/rejected": -556.9030151367188,
+      "loss": 0.598,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.7818158864974976,
+      "rewards/margins": 1.0600388050079346,
+      "rewards/rejected": -2.8418548107147217,
+      "step": 5860
+    },
+    {
+      "epoch": 0.864124834388341,
+      "grad_norm": 110.29006178066524,
+      "learning_rate": 2.7602333675218443e-08,
+      "logits/chosen": -1.392507553100586,
+      "logits/rejected": -1.0788378715515137,
+      "logps/chosen": -433.29779052734375,
+      "logps/rejected": -455.02471923828125,
+      "loss": 0.4711,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.8213212490081787,
+      "rewards/margins": 0.8197693824768066,
+      "rewards/rejected": -2.6410906314849854,
+      "step": 5870
+    },
+    {
+      "epoch": 0.8655969380244369,
+      "grad_norm": 94.93639517672632,
+      "learning_rate": 2.701842973207874e-08,
+      "logits/chosen": -1.1838589906692505,
+      "logits/rejected": -1.0877768993377686,
+      "logps/chosen": -428.2568359375,
+      "logps/rejected": -584.2926025390625,
+      "loss": 0.4963,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.5862526893615723,
+      "rewards/margins": 1.5620958805084229,
+      "rewards/rejected": -3.148348569869995,
+      "step": 5880
+    },
+    {
+      "epoch": 0.8670690416605329,
+      "grad_norm": 71.54103856862521,
+      "learning_rate": 2.6440415025064417e-08,
+      "logits/chosen": -1.4555714130401611,
+      "logits/rejected": -1.2023519277572632,
+      "logps/chosen": -458.5950622558594,
+      "logps/rejected": -593.3828125,
+      "loss": 0.5073,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -1.3977679014205933,
+      "rewards/margins": 1.7654355764389038,
+      "rewards/rejected": -3.163203477859497,
+      "step": 5890
+    },
+    {
+      "epoch": 0.8685411452966288,
+      "grad_norm": 113.61398731243004,
+      "learning_rate": 2.5868304820303027e-08,
+      "logits/chosen": -1.3414885997772217,
+      "logits/rejected": -1.0834671258926392,
+      "logps/chosen": -480.46063232421875,
+      "logps/rejected": -555.4865112304688,
+      "loss": 0.4831,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8580623865127563,
+      "rewards/margins": 1.2035136222839355,
+      "rewards/rejected": -3.0615758895874023,
+      "step": 5900
+    },
+    {
+      "epoch": 0.8700132489327249,
+      "grad_norm": 169.16449158549293,
+      "learning_rate": 2.5302114227976513e-08,
+      "logits/chosen": -1.4471521377563477,
+      "logits/rejected": -1.2723208665847778,
+      "logps/chosen": -430.6087341308594,
+      "logps/rejected": -541.4777221679688,
+      "loss": 0.3965,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.3868787288665771,
+      "rewards/margins": 1.3914029598236084,
+      "rewards/rejected": -2.7782816886901855,
+      "step": 5910
+    },
+    {
+      "epoch": 0.8714853525688209,
+      "grad_norm": 124.14812313891947,
+      "learning_rate": 2.4741858201922065e-08,
+      "logits/chosen": -1.3322300910949707,
+      "logits/rejected": -0.9847902059555054,
+      "logps/chosen": -454.60015869140625,
+      "logps/rejected": -622.1671142578125,
+      "loss": 0.465,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.8008426427841187,
+      "rewards/margins": 1.62545907497406,
+      "rewards/rejected": -3.4263014793395996,
+      "step": 5920
+    },
+    {
+      "epoch": 0.8729574562049168,
+      "grad_norm": 59.57786300290591,
+      "learning_rate": 2.4187551539237073e-08,
+      "logits/chosen": -1.3692054748535156,
+      "logits/rejected": -1.0318745374679565,
+      "logps/chosen": -430.62115478515625,
+      "logps/rejected": -508.8824768066406,
+      "loss": 0.5123,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.483989953994751,
+      "rewards/margins": 1.3129962682724,
+      "rewards/rejected": -2.7969861030578613,
+      "step": 5930
+    },
+    {
+      "epoch": 0.8744295598410128,
+      "grad_norm": 48.135899110660965,
+      "learning_rate": 2.363920887988849e-08,
+      "logits/chosen": -1.3396302461624146,
+      "logits/rejected": -1.0970475673675537,
+      "logps/chosen": -416.86474609375,
+      "logps/rejected": -642.1658935546875,
+      "loss": 0.436,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.9330217838287354,
+      "rewards/margins": 1.992821455001831,
+      "rewards/rejected": -3.9258437156677246,
+      "step": 5940
+    },
+    {
+      "epoch": 0.8759016634771087,
+      "grad_norm": 73.33774881582387,
+      "learning_rate": 2.3096844706326112e-08,
+      "logits/chosen": -1.3286715745925903,
+      "logits/rejected": -0.9419041872024536,
+      "logps/chosen": -470.03082275390625,
+      "logps/rejected": -600.4696655273438,
+      "loss": 0.4364,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -1.775228500366211,
+      "rewards/margins": 1.9548699855804443,
+      "rewards/rejected": -3.730098247528076,
+      "step": 5950
+    },
+    {
+      "epoch": 0.8773737671132048,
+      "grad_norm": 118.10841838390651,
+      "learning_rate": 2.2560473343100155e-08,
+      "logits/chosen": -1.3343017101287842,
+      "logits/rejected": -0.9438940286636353,
+      "logps/chosen": -402.0434875488281,
+      "logps/rejected": -506.3617248535156,
+      "loss": 0.5469,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -1.7077451944351196,
+      "rewards/margins": 1.385524034500122,
+      "rewards/rejected": -3.0932693481445312,
+      "step": 5960
+    },
+    {
+      "epoch": 0.8788458707493008,
+      "grad_norm": 128.7716793905253,
+      "learning_rate": 2.20301089564828e-08,
+      "logits/chosen": -1.3156306743621826,
+      "logits/rejected": -0.6910329461097717,
+      "logps/chosen": -434.342529296875,
+      "logps/rejected": -556.0278930664062,
+      "loss": 0.3873,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.5784485340118408,
+      "rewards/margins": 1.608205795288086,
+      "rewards/rejected": -3.1866543292999268,
+      "step": 5970
+    },
+    {
+      "epoch": 0.8803179743853967,
+      "grad_norm": 173.2362279170518,
+      "learning_rate": 2.150576555409392e-08,
+      "logits/chosen": -1.238092303276062,
+      "logits/rejected": -0.8196524381637573,
+      "logps/chosen": -345.23101806640625,
+      "logps/rejected": -448.9268493652344,
+      "loss": 0.5213,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.5229777097702026,
+      "rewards/margins": 1.2405390739440918,
+      "rewards/rejected": -2.763516664505005,
+      "step": 5980
+    },
+    {
+      "epoch": 0.8817900780214927,
+      "grad_norm": 107.64309048981154,
+      "learning_rate": 2.0987456984531472e-08,
+      "logits/chosen": -1.6010726690292358,
+      "logits/rejected": -1.2792376279830933,
+      "logps/chosen": -423.9590759277344,
+      "logps/rejected": -515.4378662109375,
+      "loss": 0.526,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -1.7081773281097412,
+      "rewards/margins": 1.0969882011413574,
+      "rewards/rejected": -2.8051657676696777,
+      "step": 5990
+    },
+    {
+      "epoch": 0.8832621816575887,
+      "grad_norm": 69.37673759463547,
+      "learning_rate": 2.0475196937005557e-08,
+      "logits/chosen": -1.2657551765441895,
+      "logits/rejected": -1.1628000736236572,
+      "logps/chosen": -331.22576904296875,
+      "logps/rejected": -553.7278442382812,
+      "loss": 0.4878,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -1.2025068998336792,
+      "rewards/margins": 1.6287647485733032,
+      "rewards/rejected": -2.8312716484069824,
+      "step": 6000
+    },
+    {
+      "epoch": 0.8847342852936847,
+      "grad_norm": 132.04208792710588,
+      "learning_rate": 1.9968998940976784e-08,
+      "logits/chosen": -1.4524221420288086,
+      "logits/rejected": -1.0347516536712646,
+      "logps/chosen": -428.4149475097656,
+      "logps/rejected": -584.2083740234375,
+      "loss": 0.4809,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.6654447317123413,
+      "rewards/margins": 1.8189523220062256,
+      "rewards/rejected": -3.4843971729278564,
+      "step": 6010
+    },
+    {
+      "epoch": 0.8862063889297807,
+      "grad_norm": 138.3334736509487,
+      "learning_rate": 1.9468876365799054e-08,
+      "logits/chosen": -1.1993780136108398,
+      "logits/rejected": -0.815984845161438,
+      "logps/chosen": -445.58349609375,
+      "logps/rejected": -538.29443359375,
+      "loss": 0.3615,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.7638969421386719,
+      "rewards/margins": 1.5766842365264893,
+      "rewards/rejected": -3.3405814170837402,
+      "step": 6020
+    },
+    {
+      "epoch": 0.8876784925658766,
+      "grad_norm": 87.43403828640025,
+      "learning_rate": 1.89748424203664e-08,
+      "logits/chosen": -1.4031301736831665,
+      "logits/rejected": -0.9359944462776184,
+      "logps/chosen": -467.64886474609375,
+      "logps/rejected": -519.8732299804688,
+      "loss": 0.437,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.5854235887527466,
+      "rewards/margins": 1.0862160921096802,
+      "rewards/rejected": -2.6716394424438477,
+      "step": 6030
+    },
+    {
+      "epoch": 0.8891505962019726,
+      "grad_norm": 75.77775039688505,
+      "learning_rate": 1.8486910152764102e-08,
+      "logits/chosen": -1.2027417421340942,
+      "logits/rejected": -0.9280147552490234,
+      "logps/chosen": -422.48388671875,
+      "logps/rejected": -555.5831298828125,
+      "loss": 0.4267,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.6530888080596924,
+      "rewards/margins": 1.3602993488311768,
+      "rewards/rejected": -3.013388156890869,
+      "step": 6040
+    },
+    {
+      "epoch": 0.8906226998380686,
+      "grad_norm": 130.35340754132244,
+      "learning_rate": 1.8005092449924292e-08,
+      "logits/chosen": -1.3218846321105957,
+      "logits/rejected": -1.231888771057129,
+      "logps/chosen": -389.1402587890625,
+      "logps/rejected": -465.5093688964844,
+      "loss": 0.5728,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.763266921043396,
+      "rewards/margins": 0.736245334148407,
+      "rewards/rejected": -2.499511957168579,
+      "step": 6050
+    },
+    {
+      "epoch": 0.8920948034741646,
+      "grad_norm": 190.32137629404687,
+      "learning_rate": 1.7529402037285312e-08,
+      "logits/chosen": -1.3747310638427734,
+      "logits/rejected": -1.2134448289871216,
+      "logps/chosen": -437.971435546875,
+      "logps/rejected": -612.3836059570312,
+      "loss": 0.4591,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.6321361064910889,
+      "rewards/margins": 1.8617744445800781,
+      "rewards/rejected": -3.493910551071167,
+      "step": 6060
+    },
+    {
+      "epoch": 0.8935669071102605,
+      "grad_norm": 72.90205062854201,
+      "learning_rate": 1.7059851478455594e-08,
+      "logits/chosen": -1.336159348487854,
+      "logits/rejected": -1.0463287830352783,
+      "logps/chosen": -447.993896484375,
+      "logps/rejected": -550.3836059570312,
+      "loss": 0.4165,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.861944556236267,
+      "rewards/margins": 1.318798303604126,
+      "rewards/rejected": -3.1807429790496826,
+      "step": 6070
+    },
+    {
+      "epoch": 0.8950390107463565,
+      "grad_norm": 98.9872237330582,
+      "learning_rate": 1.6596453174882145e-08,
+      "logits/chosen": -1.0017184019088745,
+      "logits/rejected": -1.0421884059906006,
+      "logps/chosen": -399.8570251464844,
+      "logps/rejected": -596.8233642578125,
+      "loss": 0.3481,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -2.0980629920959473,
+      "rewards/margins": 1.2766414880752563,
+      "rewards/rejected": -3.374704360961914,
+      "step": 6080
+    },
+    {
+      "epoch": 0.8965111143824526,
+      "grad_norm": 59.48426601433193,
+      "learning_rate": 1.6139219365522732e-08,
+      "logits/chosen": -1.4107369184494019,
+      "logits/rejected": -1.1122539043426514,
+      "logps/chosen": -514.1741943359375,
+      "logps/rejected": -623.8187866210938,
+      "loss": 0.4501,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.7783997058868408,
+      "rewards/margins": 1.2535008192062378,
+      "rewards/rejected": -3.0319008827209473,
+      "step": 6090
+    },
+    {
+      "epoch": 0.8979832180185485,
+      "grad_norm": 100.29092453904826,
+      "learning_rate": 1.5688162126522853e-08,
+      "logits/chosen": -1.4325015544891357,
+      "logits/rejected": -0.9693012237548828,
+      "logps/chosen": -461.904541015625,
+      "logps/rejected": -509.4136657714844,
+      "loss": 0.467,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.8348863124847412,
+      "rewards/margins": 1.2343018054962158,
+      "rewards/rejected": -3.069188356399536,
+      "step": 6100
+    },
+    {
+      "epoch": 0.8994553216546445,
+      "grad_norm": 84.13420698485271,
+      "learning_rate": 1.5243293370896554e-08,
+      "logits/chosen": -1.4444586038589478,
+      "logits/rejected": -1.0995537042617798,
+      "logps/chosen": -440.20892333984375,
+      "logps/rejected": -510.4673767089844,
+      "loss": 0.4906,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -1.6723476648330688,
+      "rewards/margins": 0.9577652215957642,
+      "rewards/rejected": -2.630113124847412,
+      "step": 6110
+    },
+    {
+      "epoch": 0.9009274252907404,
+      "grad_norm": 88.46133023122071,
+      "learning_rate": 1.4804624848211933e-08,
+      "logits/chosen": -1.1499179601669312,
+      "logits/rejected": -0.7655227184295654,
+      "logps/chosen": -420.9551696777344,
+      "logps/rejected": -512.953125,
+      "loss": 0.55,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8270105123519897,
+      "rewards/margins": 0.7387310266494751,
+      "rewards/rejected": -2.5657413005828857,
+      "step": 6120
+    },
+    {
+      "epoch": 0.9023995289268365,
+      "grad_norm": 112.99296743675342,
+      "learning_rate": 1.4372168144280744e-08,
+      "logits/chosen": -1.182217001914978,
+      "logits/rejected": -0.7810528874397278,
+      "logps/chosen": -432.20880126953125,
+      "logps/rejected": -523.0076904296875,
+      "loss": 0.5415,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.7092154026031494,
+      "rewards/margins": 1.2686071395874023,
+      "rewards/rejected": -2.9778225421905518,
+      "step": 6130
+    },
+    {
+      "epoch": 0.9038716325629325,
+      "grad_norm": 121.96728716275769,
+      "learning_rate": 1.3945934680852617e-08,
+      "logits/chosen": -1.392909288406372,
+      "logits/rejected": -0.8278254270553589,
+      "logps/chosen": -475.64398193359375,
+      "logps/rejected": -598.2657470703125,
+      "loss": 0.4422,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.8078081607818604,
+      "rewards/margins": 1.7844040393829346,
+      "rewards/rejected": -3.592212200164795,
+      "step": 6140
+    },
+    {
+      "epoch": 0.9053437361990284,
+      "grad_norm": 72.06405639321714,
+      "learning_rate": 1.3525935715313074e-08,
+      "logits/chosen": -1.1380715370178223,
+      "logits/rejected": -0.898354172706604,
+      "logps/chosen": -381.2165832519531,
+      "logps/rejected": -534.9982299804688,
+      "loss": 0.4306,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.9703505039215088,
+      "rewards/margins": 1.1985515356063843,
+      "rewards/rejected": -3.1689019203186035,
+      "step": 6150
+    },
+    {
+      "epoch": 0.9068158398351244,
+      "grad_norm": 108.29273236635373,
+      "learning_rate": 1.3112182340386412e-08,
+      "logits/chosen": -1.0974479913711548,
+      "logits/rejected": -0.8271905779838562,
+      "logps/chosen": -378.52362060546875,
+      "logps/rejected": -518.6552734375,
+      "loss": 0.4617,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.7146676778793335,
+      "rewards/margins": 1.60419499874115,
+      "rewards/rejected": -3.3188629150390625,
+      "step": 6160
+    },
+    {
+      "epoch": 0.9082879434712203,
+      "grad_norm": 101.71504234754592,
+      "learning_rate": 1.2704685483842626e-08,
+      "logits/chosen": -1.2984583377838135,
+      "logits/rejected": -0.7792263627052307,
+      "logps/chosen": -352.86566162109375,
+      "logps/rejected": -436.63922119140625,
+      "loss": 0.525,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.6000096797943115,
+      "rewards/margins": 1.2597168684005737,
+      "rewards/rejected": -2.859726667404175,
+      "step": 6170
+    },
+    {
+      "epoch": 0.9097600471073164,
+      "grad_norm": 93.0214661593294,
+      "learning_rate": 1.2303455908208915e-08,
+      "logits/chosen": -1.4515659809112549,
+      "logits/rejected": -1.148747205734253,
+      "logps/chosen": -378.54266357421875,
+      "logps/rejected": -454.61065673828125,
+      "loss": 0.4494,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.527250051498413,
+      "rewards/margins": 1.2347928285598755,
+      "rewards/rejected": -2.76204252243042,
+      "step": 6180
+    },
+    {
+      "epoch": 0.9112321507434124,
+      "grad_norm": 78.22321116231281,
+      "learning_rate": 1.1908504210485337e-08,
+      "logits/chosen": -1.262365460395813,
+      "logits/rejected": -0.6896898150444031,
+      "logps/chosen": -477.6587829589844,
+      "logps/rejected": -525.4725341796875,
+      "loss": 0.4674,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.6702487468719482,
+      "rewards/margins": 1.1898547410964966,
+      "rewards/rejected": -2.860103130340576,
+      "step": 6190
+    },
+    {
+      "epoch": 0.9127042543795083,
+      "grad_norm": 68.9366431319535,
+      "learning_rate": 1.1519840821864968e-08,
+      "logits/chosen": -1.3926799297332764,
+      "logits/rejected": -1.010058879852295,
+      "logps/chosen": -483.1468200683594,
+      "logps/rejected": -593.4761352539062,
+      "loss": 0.4451,
+      "rewards/accuracies": 0.7000000476837158,
+      "rewards/chosen": -1.7724260091781616,
+      "rewards/margins": 1.0586087703704834,
+      "rewards/rejected": -2.8310346603393555,
+      "step": 6200
+    },
+    {
+      "epoch": 0.9141763580156043,
+      "grad_norm": 255.98270815188275,
+      "learning_rate": 1.113747600745829e-08,
+      "logits/chosen": -1.3348357677459717,
+      "logits/rejected": -1.1368389129638672,
+      "logps/chosen": -411.146728515625,
+      "logps/rejected": -558.9170532226562,
+      "loss": 0.4938,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.572711706161499,
+      "rewards/margins": 1.3852193355560303,
+      "rewards/rejected": -2.9579310417175293,
+      "step": 6210
+    },
+    {
+      "epoch": 0.9156484616517003,
+      "grad_norm": 171.3237826710108,
+      "learning_rate": 1.0761419866022247e-08,
+      "logits/chosen": -1.5173606872558594,
+      "logits/rejected": -1.3542439937591553,
+      "logps/chosen": -396.34625244140625,
+      "logps/rejected": -472.72344970703125,
+      "loss": 0.5572,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.4570844173431396,
+      "rewards/margins": 1.0616931915283203,
+      "rewards/rejected": -2.518777370452881,
+      "step": 6220
+    },
+    {
+      "epoch": 0.9171205652877963,
+      "grad_norm": 96.76388140858792,
+      "learning_rate": 1.039168232969348e-08,
+      "logits/chosen": -1.2802958488464355,
+      "logits/rejected": -1.0718050003051758,
+      "logps/chosen": -438.5003967285156,
+      "logps/rejected": -513.9151611328125,
+      "loss": 0.5158,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.7826290130615234,
+      "rewards/margins": 0.9541376233100891,
+      "rewards/rejected": -2.736766815185547,
+      "step": 6230
+    },
+    {
+      "epoch": 0.9185926689238922,
+      "grad_norm": 208.65795799594645,
+      "learning_rate": 1.0028273163725898e-08,
+      "logits/chosen": -1.3111271858215332,
+      "logits/rejected": -1.194875955581665,
+      "logps/chosen": -519.2174072265625,
+      "logps/rejected": -596.43408203125,
+      "loss": 0.4112,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -1.882115364074707,
+      "rewards/margins": 1.3816559314727783,
+      "rewards/rejected": -3.2637715339660645,
+      "step": 6240
+    },
+    {
+      "epoch": 0.9200647725599882,
+      "grad_norm": 110.28853386869616,
+      "learning_rate": 9.671201966232889e-09,
+      "logits/chosen": -1.2120047807693481,
+      "logits/rejected": -1.106927514076233,
+      "logps/chosen": -417.23101806640625,
+      "logps/rejected": -551.600341796875,
+      "loss": 0.4188,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.9461908340454102,
+      "rewards/margins": 1.2960798740386963,
+      "rewards/rejected": -3.2422707080841064,
+      "step": 6250
+    },
+    {
+      "epoch": 0.9215368761960842,
+      "grad_norm": 65.0103326745241,
+      "learning_rate": 9.320478167933659e-09,
+      "logits/chosen": -1.3543187379837036,
+      "logits/rejected": -1.2014297246932983,
+      "logps/chosen": -451.106689453125,
+      "logps/rejected": -591.7469482421875,
+      "loss": 0.4751,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.7379400730133057,
+      "rewards/margins": 1.395159125328064,
+      "rewards/rejected": -3.13309907913208,
+      "step": 6260
+    },
+    {
+      "epoch": 0.9230089798321802,
+      "grad_norm": 136.73147505961566,
+      "learning_rate": 8.976111031904377e-09,
+      "logits/chosen": -1.7549540996551514,
+      "logits/rejected": -1.2522004842758179,
+      "logps/chosen": -467.4527893066406,
+      "logps/rejected": -527.7576904296875,
+      "loss": 0.433,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -1.672555923461914,
+      "rewards/margins": 1.3347749710083008,
+      "rewards/rejected": -3.0073306560516357,
+      "step": 6270
+    },
+    {
+      "epoch": 0.9244810834682762,
+      "grad_norm": 197.32138066123477,
+      "learning_rate": 8.638109653333419e-09,
+      "logits/chosen": -1.5915447473526,
+      "logits/rejected": -1.028139591217041,
+      "logps/chosen": -544.760498046875,
+      "logps/rejected": -612.6495361328125,
+      "loss": 0.4717,
+      "rewards/accuracies": 0.8666666150093079,
+      "rewards/chosen": -1.9581520557403564,
+      "rewards/margins": 1.572818398475647,
+      "rewards/rejected": -3.530970811843872,
+      "step": 6280
+    },
+    {
+      "epoch": 0.9259531871043721,
+      "grad_norm": 88.9975140530782,
+      "learning_rate": 8.306482959281103e-09,
+      "logits/chosen": -1.1016359329223633,
+      "logits/rejected": -0.9385136365890503,
+      "logps/chosen": -415.51953125,
+      "logps/rejected": -483.88916015625,
+      "loss": 0.5574,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -2.1461730003356934,
+      "rewards/margins": 0.9563619494438171,
+      "rewards/rejected": -3.102534770965576,
+      "step": 6290
+    },
+    {
+      "epoch": 0.9274252907404681,
+      "grad_norm": 136.91687710205173,
+      "learning_rate": 7.981239708443971e-09,
+      "logits/chosen": -1.3256946802139282,
+      "logits/rejected": -0.9548309445381165,
+      "logps/chosen": -456.28167724609375,
+      "logps/rejected": -578.2469482421875,
+      "loss": 0.5347,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.6343233585357666,
+      "rewards/margins": 1.664646863937378,
+      "rewards/rejected": -3.2989699840545654,
+      "step": 6300
+    },
+    {
+      "epoch": 0.9288973943765642,
+      "grad_norm": 141.73391501968828,
+      "learning_rate": 7.662388490923378e-09,
+      "logits/chosen": -1.2113536596298218,
+      "logits/rejected": -0.9762833714485168,
+      "logps/chosen": -510.31536865234375,
+      "logps/rejected": -675.57958984375,
+      "loss": 0.4457,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -2.343094825744629,
+      "rewards/margins": 1.4834887981414795,
+      "rewards/rejected": -3.8265833854675293,
+      "step": 6310
+    },
+    {
+      "epoch": 0.9303694980126601,
+      "grad_norm": 95.39226728017128,
+      "learning_rate": 7.3499377279988e-09,
+      "logits/chosen": -1.3638355731964111,
+      "logits/rejected": -1.0438239574432373,
+      "logps/chosen": -456.48736572265625,
+      "logps/rejected": -506.72283935546875,
+      "loss": 0.6138,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -2.035297155380249,
+      "rewards/margins": 1.0015877485275269,
+      "rewards/rejected": -3.0368847846984863,
+      "step": 6320
+    },
+    {
+      "epoch": 0.9318416016487561,
+      "grad_norm": 83.18375545076209,
+      "learning_rate": 7.043895671905248e-09,
+      "logits/chosen": -1.3861024379730225,
+      "logits/rejected": -1.381811499595642,
+      "logps/chosen": -401.227783203125,
+      "logps/rejected": -576.5643920898438,
+      "loss": 0.4281,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.6423763036727905,
+      "rewards/margins": 1.475299596786499,
+      "rewards/rejected": -3.117676019668579,
+      "step": 6330
+    },
+    {
+      "epoch": 0.933313705284852,
+      "grad_norm": 157.54591394824698,
+      "learning_rate": 6.744270405615371e-09,
+      "logits/chosen": -1.3402183055877686,
+      "logits/rejected": -1.0885117053985596,
+      "logps/chosen": -445.0760803222656,
+      "logps/rejected": -595.1424560546875,
+      "loss": 0.4372,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.563539743423462,
+      "rewards/margins": 2.049320936203003,
+      "rewards/rejected": -3.612860918045044,
+      "step": 6340
+    },
+    {
+      "epoch": 0.934785808920948,
+      "grad_norm": 78.5332718700823,
+      "learning_rate": 6.4510698426259165e-09,
+      "logits/chosen": -1.3075374364852905,
+      "logits/rejected": -1.0480823516845703,
+      "logps/chosen": -421.25421142578125,
+      "logps/rejected": -520.3898315429688,
+      "loss": 0.3731,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.5807876586914062,
+      "rewards/margins": 1.3279130458831787,
+      "rewards/rejected": -2.908700704574585,
+      "step": 6350
+    },
+    {
+      "epoch": 0.936257912557044,
+      "grad_norm": 110.06828598299056,
+      "learning_rate": 6.164301726748838e-09,
+      "logits/chosen": -1.358068823814392,
+      "logits/rejected": -0.9059964418411255,
+      "logps/chosen": -401.7532043457031,
+      "logps/rejected": -521.3616333007812,
+      "loss": 0.4533,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -1.3470642566680908,
+      "rewards/margins": 1.7689765691757202,
+      "rewards/rejected": -3.1160407066345215,
+      "step": 6360
+    },
+    {
+      "epoch": 0.93773001619314,
+      "grad_norm": 153.47649978619683,
+      "learning_rate": 5.883973631906736e-09,
+      "logits/chosen": -1.3301823139190674,
+      "logits/rejected": -1.1308788061141968,
+      "logps/chosen": -468.0716857910156,
+      "logps/rejected": -622.98291015625,
+      "loss": 0.4996,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -2.2261788845062256,
+      "rewards/margins": 1.54001784324646,
+      "rewards/rejected": -3.7661972045898438,
+      "step": 6370
+    },
+    {
+      "epoch": 0.939202119829236,
+      "grad_norm": 85.53426929713154,
+      "learning_rate": 5.610092961932766e-09,
+      "logits/chosen": -1.3597456216812134,
+      "logits/rejected": -1.3059452772140503,
+      "logps/chosen": -460.44781494140625,
+      "logps/rejected": -603.2689208984375,
+      "loss": 0.5525,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.913364052772522,
+      "rewards/margins": 1.6412509679794312,
+      "rewards/rejected": -3.554615020751953,
+      "step": 6380
+    },
+    {
+      "epoch": 0.9406742234653319,
+      "grad_norm": 96.2904126112998,
+      "learning_rate": 5.3426669503751355e-09,
+      "logits/chosen": -1.5512850284576416,
+      "logits/rejected": -0.9863389730453491,
+      "logps/chosen": -525.7786254882812,
+      "logps/rejected": -536.9043579101562,
+      "loss": 0.4689,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -2.06516695022583,
+      "rewards/margins": 0.9437375068664551,
+      "rewards/rejected": -3.008904457092285,
+      "step": 6390
+    },
+    {
+      "epoch": 0.942146327101428,
+      "grad_norm": 142.36380621173555,
+      "learning_rate": 5.081702660306025e-09,
+      "logits/chosen": -1.478801965713501,
+      "logits/rejected": -0.8947819471359253,
+      "logps/chosen": -473.7356872558594,
+      "logps/rejected": -640.308837890625,
+      "loss": 0.5642,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -2.126918315887451,
+      "rewards/margins": 1.6010997295379639,
+      "rewards/rejected": -3.728017807006836,
+      "step": 6400
+    },
+    {
+      "epoch": 0.9436184307375239,
+      "grad_norm": 152.4371615430786,
+      "learning_rate": 4.827206984135024e-09,
+      "logits/chosen": -1.2242169380187988,
+      "logits/rejected": -1.00106942653656,
+      "logps/chosen": -406.49755859375,
+      "logps/rejected": -521.4188842773438,
+      "loss": 0.5218,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -2.009997844696045,
+      "rewards/margins": 1.1500599384307861,
+      "rewards/rejected": -3.160057544708252,
+      "step": 6410
+    },
+    {
+      "epoch": 0.9450905343736199,
+      "grad_norm": 109.5170505315079,
+      "learning_rate": 4.579186643427241e-09,
+      "logits/chosen": -1.4265880584716797,
+      "logits/rejected": -1.1079994440078735,
+      "logps/chosen": -517.7324829101562,
+      "logps/rejected": -550.4271240234375,
+      "loss": 0.5119,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.8106954097747803,
+      "rewards/margins": 1.1361420154571533,
+      "rewards/rejected": -2.9468376636505127,
+      "step": 6420
+    },
+    {
+      "epoch": 0.9465626380097159,
+      "grad_norm": 120.06199176900785,
+      "learning_rate": 4.337648188725618e-09,
+      "logits/chosen": -1.2486460208892822,
+      "logits/rejected": -0.9750927686691284,
+      "logps/chosen": -422.97454833984375,
+      "logps/rejected": -530.0245971679688,
+      "loss": 0.5378,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.3605560064315796,
+      "rewards/margins": 1.57229745388031,
+      "rewards/rejected": -2.9328534603118896,
+      "step": 6430
+    },
+    {
+      "epoch": 0.9480347416458119,
+      "grad_norm": 121.72026440077309,
+      "learning_rate": 4.102597999377927e-09,
+      "logits/chosen": -1.3509511947631836,
+      "logits/rejected": -1.1400359869003296,
+      "logps/chosen": -549.1176147460938,
+      "logps/rejected": -692.47509765625,
+      "loss": 0.4896,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -2.322995662689209,
+      "rewards/margins": 1.242288589477539,
+      "rewards/rejected": -3.565284013748169,
+      "step": 6440
+    },
+    {
+      "epoch": 0.9495068452819079,
+      "grad_norm": 221.28721364142348,
+      "learning_rate": 3.874042283368406e-09,
+      "logits/chosen": -1.3209785223007202,
+      "logits/rejected": -1.0659959316253662,
+      "logps/chosen": -450.44146728515625,
+      "logps/rejected": -583.3470458984375,
+      "loss": 0.4932,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -2.1836166381835938,
+      "rewards/margins": 1.1096569299697876,
+      "rewards/rejected": -3.2932732105255127,
+      "step": 6450
+    },
+    {
+      "epoch": 0.9509789489180038,
+      "grad_norm": 100.69938687648991,
+      "learning_rate": 3.6519870771536952e-09,
+      "logits/chosen": -1.286990761756897,
+      "logits/rejected": -1.3209233283996582,
+      "logps/chosen": -437.3722229003906,
+      "logps/rejected": -582.5029296875,
+      "loss": 0.4342,
+      "rewards/accuracies": 0.6999999284744263,
+      "rewards/chosen": -1.9360355138778687,
+      "rewards/margins": 0.7741468548774719,
+      "rewards/rejected": -2.7101826667785645,
+      "step": 6460
+    },
+    {
+      "epoch": 0.9524510525540998,
+      "grad_norm": 113.24796094369674,
+      "learning_rate": 3.436438245503409e-09,
+      "logits/chosen": -1.1490650177001953,
+      "logits/rejected": -1.061572790145874,
+      "logps/chosen": -461.55877685546875,
+      "logps/rejected": -567.5509033203125,
+      "loss": 0.558,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.6923081874847412,
+      "rewards/margins": 1.226409673690796,
+      "rewards/rejected": -2.918717861175537,
+      "step": 6470
+    },
+    {
+      "epoch": 0.9539231561901957,
+      "grad_norm": 79.79596970660135,
+      "learning_rate": 3.2274014813452875e-09,
+      "logits/chosen": -1.1326738595962524,
+      "logits/rejected": -0.8211013078689575,
+      "logps/chosen": -428.6504821777344,
+      "logps/rejected": -677.7749633789062,
+      "loss": 0.4101,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.111638307571411,
+      "rewards/margins": 1.9096629619598389,
+      "rewards/rejected": -4.021300792694092,
+      "step": 6480
+    },
+    {
+      "epoch": 0.9553952598262918,
+      "grad_norm": 125.84258421914738,
+      "learning_rate": 3.0248823056148175e-09,
+      "logits/chosen": -1.1957733631134033,
+      "logits/rejected": -1.4037882089614868,
+      "logps/chosen": -433.4906311035156,
+      "logps/rejected": -587.3173217773438,
+      "loss": 0.4584,
+      "rewards/accuracies": 0.73333340883255,
+      "rewards/chosen": -1.8290573358535767,
+      "rewards/margins": 1.4790170192718506,
+      "rewards/rejected": -3.3080742359161377,
+      "step": 6490
+    },
+    {
+      "epoch": 0.9568673634623878,
+      "grad_norm": 150.38562504197765,
+      "learning_rate": 2.828886067109404e-09,
+      "logits/chosen": -1.4119644165039062,
+      "logits/rejected": -1.2277817726135254,
+      "logps/chosen": -508.92205810546875,
+      "logps/rejected": -642.3756103515625,
+      "loss": 0.4982,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -2.104766368865967,
+      "rewards/margins": 1.6219040155410767,
+      "rewards/rejected": -3.7266697883605957,
+      "step": 6500
+    },
+    {
+      "epoch": 0.9583394670984837,
+      "grad_norm": 98.17333195909225,
+      "learning_rate": 2.639417942347122e-09,
+      "logits/chosen": -1.3974770307540894,
+      "logits/rejected": -1.0616443157196045,
+      "logps/chosen": -366.06060791015625,
+      "logps/rejected": -589.9656982421875,
+      "loss": 0.5112,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.51221764087677,
+      "rewards/margins": 2.0129287242889404,
+      "rewards/rejected": -3.525146007537842,
+      "step": 6510
+    },
+    {
+      "epoch": 0.9598115707345797,
+      "grad_norm": 187.6051838583204,
+      "learning_rate": 2.456482935429993e-09,
+      "logits/chosen": -1.3879002332687378,
+      "logits/rejected": -1.0190473794937134,
+      "logps/chosen": -480.7325744628906,
+      "logps/rejected": -579.7218017578125,
+      "loss": 0.6549,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -2.0841310024261475,
+      "rewards/margins": 1.207476258277893,
+      "rewards/rejected": -3.291607618331909,
+      "step": 6520
+    },
+    {
+      "epoch": 0.9612836743706757,
+      "grad_norm": 87.85131668882238,
+      "learning_rate": 2.2800858779118417e-09,
+      "logits/chosen": -1.238096833229065,
+      "logits/rejected": -1.061269760131836,
+      "logps/chosen": -429.9205017089844,
+      "logps/rejected": -571.918212890625,
+      "loss": 0.6518,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.7396520376205444,
+      "rewards/margins": 1.5972769260406494,
+      "rewards/rejected": -3.336928606033325,
+      "step": 6530
+    },
+    {
+      "epoch": 0.9627557780067717,
+      "grad_norm": 122.8780597463822,
+      "learning_rate": 2.1102314286705334e-09,
+      "logits/chosen": -1.3741991519927979,
+      "logits/rejected": -1.2921290397644043,
+      "logps/chosen": -510.30975341796875,
+      "logps/rejected": -513.4083251953125,
+      "loss": 0.486,
+      "rewards/accuracies": 0.5666666626930237,
+      "rewards/chosen": -1.8251821994781494,
+      "rewards/margins": 0.5057021379470825,
+      "rewards/rejected": -2.3308842182159424,
+      "step": 6540
+    },
+    {
+      "epoch": 0.9642278816428677,
+      "grad_norm": 91.14450887378497,
+      "learning_rate": 1.9469240737852445e-09,
+      "logits/chosen": -1.2866744995117188,
+      "logits/rejected": -1.1259269714355469,
+      "logps/chosen": -322.0516357421875,
+      "logps/rejected": -511.1654357910156,
+      "loss": 0.4444,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.310871958732605,
+      "rewards/margins": 1.750274419784546,
+      "rewards/rejected": -3.0611460208892822,
+      "step": 6550
+    },
+    {
+      "epoch": 0.9656999852789636,
+      "grad_norm": 89.35137950308767,
+      "learning_rate": 1.790168126417635e-09,
+      "logits/chosen": -1.3406840562820435,
+      "logits/rejected": -0.8247464895248413,
+      "logps/chosen": -352.471923828125,
+      "logps/rejected": -486.70989990234375,
+      "loss": 0.6235,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.3377752304077148,
+      "rewards/margins": 1.5931618213653564,
+      "rewards/rejected": -2.9309370517730713,
+      "step": 6560
+    },
+    {
+      "epoch": 0.9671720889150596,
+      "grad_norm": 91.35928129358597,
+      "learning_rate": 1.6399677266982214e-09,
+      "logits/chosen": -1.3589528799057007,
+      "logits/rejected": -1.1516953706741333,
+      "logps/chosen": -499.2710876464844,
+      "logps/rejected": -571.6011962890625,
+      "loss": 0.4366,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0111379623413086,
+      "rewards/margins": 1.0724924802780151,
+      "rewards/rejected": -3.0836308002471924,
+      "step": 6570
+    },
+    {
+      "epoch": 0.9686441925511556,
+      "grad_norm": 243.42171472032464,
+      "learning_rate": 1.4963268416167685e-09,
+      "logits/chosen": -1.2164876461029053,
+      "logits/rejected": -0.8679919242858887,
+      "logps/chosen": -421.5487365722656,
+      "logps/rejected": -547.1917724609375,
+      "loss": 0.5454,
+      "rewards/accuracies": 0.7333333492279053,
+      "rewards/chosen": -1.509100079536438,
+      "rewards/margins": 1.6187273263931274,
+      "rewards/rejected": -3.1278271675109863,
+      "step": 6580
+    },
+    {
+      "epoch": 0.9701162961872516,
+      "grad_norm": 90.16390237464518,
+      "learning_rate": 1.3592492649177056e-09,
+      "logits/chosen": -1.0717471837997437,
+      "logits/rejected": -0.9164643287658691,
+      "logps/chosen": -423.8712463378906,
+      "logps/rejected": -534.1849365234375,
+      "loss": 0.4693,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -1.6814769506454468,
+      "rewards/margins": 1.592142105102539,
+      "rewards/rejected": -3.2736194133758545,
+      "step": 6590
+    },
+    {
+      "epoch": 0.9715883998233475,
+      "grad_norm": 167.80484523337927,
+      "learning_rate": 1.2287386169998747e-09,
+      "logits/chosen": -1.6180499792099,
+      "logits/rejected": -1.0677508115768433,
+      "logps/chosen": -430.90753173828125,
+      "logps/rejected": -562.3106079101562,
+      "loss": 0.4464,
+      "rewards/accuracies": 0.7999999523162842,
+      "rewards/chosen": -1.796453833580017,
+      "rewards/margins": 1.643140435218811,
+      "rewards/rejected": -3.439594268798828,
+      "step": 6600
+    },
+    {
+      "epoch": 0.9730605034594435,
+      "grad_norm": 121.2607288611665,
+      "learning_rate": 1.1047983448209397e-09,
+      "logits/chosen": -1.3967403173446655,
+      "logits/rejected": -1.1385594606399536,
+      "logps/chosen": -437.872802734375,
+      "logps/rejected": -550.6277465820312,
+      "loss": 0.4394,
+      "rewards/accuracies": 0.9000000953674316,
+      "rewards/chosen": -1.5407131910324097,
+      "rewards/margins": 1.4084869623184204,
+      "rewards/rejected": -2.94920015335083,
+      "step": 6610
+    },
+    {
+      "epoch": 0.9745326070955396,
+      "grad_norm": 62.21049188639438,
+      "learning_rate": 9.87431721806209e-10,
+      "logits/chosen": -1.5890763998031616,
+      "logits/rejected": -1.2994145154953003,
+      "logps/chosen": -421.8890686035156,
+      "logps/rejected": -507.55224609375,
+      "loss": 0.4199,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.5588992834091187,
+      "rewards/margins": 1.3105344772338867,
+      "rewards/rejected": -2.869433879852295,
+      "step": 6620
+    },
+    {
+      "epoch": 0.9760047107316355,
+      "grad_norm": 63.232338001046614,
+      "learning_rate": 8.766418477623716e-10,
+      "logits/chosen": -1.2572543621063232,
+      "logits/rejected": -0.9364638328552246,
+      "logps/chosen": -413.59002685546875,
+      "logps/rejected": -499.2174377441406,
+      "loss": 0.3867,
+      "rewards/accuracies": 0.7666667103767395,
+      "rewards/chosen": -1.4785820245742798,
+      "rewards/margins": 1.3318545818328857,
+      "rewards/rejected": -2.810436725616455,
+      "step": 6630
+    },
+    {
+      "epoch": 0.9774768143677315,
+      "grad_norm": 157.68941789773905,
+      "learning_rate": 7.724316487954796e-10,
+      "logits/chosen": -1.3735524415969849,
+      "logits/rejected": -1.0742418766021729,
+      "logps/chosen": -404.0660400390625,
+      "logps/rejected": -567.3262939453125,
+      "loss": 0.5536,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -1.6344239711761475,
+      "rewards/margins": 1.5732135772705078,
+      "rewards/rejected": -3.207637310028076,
+      "step": 6640
+    },
+    {
+      "epoch": 0.9789489180038274,
+      "grad_norm": 127.63106630389774,
+      "learning_rate": 6.748038772337595e-10,
+      "logits/chosen": -1.3165947198867798,
+      "logits/rejected": -0.9859461784362793,
+      "logps/chosen": -435.93133544921875,
+      "logps/rejected": -533.8653564453125,
+      "loss": 0.5505,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.759289026260376,
+      "rewards/margins": 1.2258495092391968,
+      "rewards/rejected": -2.985138416290283,
+      "step": 6650
+    },
+    {
+      "epoch": 0.9804210216399234,
+      "grad_norm": 176.17770544447473,
+      "learning_rate": 5.837611115549201e-10,
+      "logits/chosen": -1.5459668636322021,
+      "logits/rejected": -0.888058066368103,
+      "logps/chosen": -464.20172119140625,
+      "logps/rejected": -534.7344970703125,
+      "loss": 0.5347,
+      "rewards/accuracies": 0.7333332896232605,
+      "rewards/chosen": -1.4714539051055908,
+      "rewards/margins": 1.4597190618515015,
+      "rewards/rejected": -2.9311728477478027,
+      "step": 6660
+    },
+    {
+      "epoch": 0.9818931252760195,
+      "grad_norm": 166.41130513975992,
+      "learning_rate": 4.993057563179303e-10,
+      "logits/chosen": -1.5793631076812744,
+      "logits/rejected": -1.1331535577774048,
+      "logps/chosen": -499.2445373535156,
+      "logps/rejected": -556.5195922851562,
+      "loss": 0.3825,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.7106945514678955,
+      "rewards/margins": 1.5195393562316895,
+      "rewards/rejected": -3.230233669281006,
+      "step": 6670
+    },
+    {
+      "epoch": 0.9833652289121154,
+      "grad_norm": 236.0814983966395,
+      "learning_rate": 4.2144004209962445e-10,
+      "logits/chosen": -1.285900592803955,
+      "logits/rejected": -1.2104876041412354,
+      "logps/chosen": -454.2847595214844,
+      "logps/rejected": -510.22247314453125,
+      "loss": 0.4775,
+      "rewards/accuracies": 0.6333333253860474,
+      "rewards/chosen": -1.477925419807434,
+      "rewards/margins": 0.8418137431144714,
+      "rewards/rejected": -2.3197388648986816,
+      "step": 6680
+    },
+    {
+      "epoch": 0.9848373325482114,
+      "grad_norm": 68.83875850322735,
+      "learning_rate": 3.501660254358607e-10,
+      "logits/chosen": -1.2782337665557861,
+      "logits/rejected": -0.8187475204467773,
+      "logps/chosen": -432.896728515625,
+      "logps/rejected": -539.8255615234375,
+      "loss": 0.4564,
+      "rewards/accuracies": 0.6666666865348816,
+      "rewards/chosen": -2.147003650665283,
+      "rewards/margins": 1.3009148836135864,
+      "rewards/rejected": -3.447918653488159,
+      "step": 6690
+    },
+    {
+      "epoch": 0.9863094361843073,
+      "grad_norm": 67.97220529141755,
+      "learning_rate": 2.854855887669538e-10,
+      "logits/chosen": -1.3216626644134521,
+      "logits/rejected": -1.440431833267212,
+      "logps/chosen": -487.4225158691406,
+      "logps/rejected": -620.2841796875,
+      "loss": 0.4624,
+      "rewards/accuracies": 0.73333340883255,
+      "rewards/chosen": -1.8056331872940063,
+      "rewards/margins": 1.089308738708496,
+      "rewards/rejected": -2.894942045211792,
+      "step": 6700
+    },
+    {
+      "epoch": 0.9877815398204034,
+      "grad_norm": 121.32963259885697,
+      "learning_rate": 2.274004403882146e-10,
+      "logits/chosen": -1.367511510848999,
+      "logits/rejected": -1.0503766536712646,
+      "logps/chosen": -444.0887756347656,
+      "logps/rejected": -604.0455322265625,
+      "loss": 0.4112,
+      "rewards/accuracies": 0.76666659116745,
+      "rewards/chosen": -1.7945635318756104,
+      "rewards/margins": 1.5105736255645752,
+      "rewards/rejected": -3.3051364421844482,
+      "step": 6710
+    },
+    {
+      "epoch": 0.9892536434564994,
+      "grad_norm": 142.330296107857,
+      "learning_rate": 1.7591211440468047e-10,
+      "logits/chosen": -1.2823817729949951,
+      "logits/rejected": -0.941415011882782,
+      "logps/chosen": -417.048095703125,
+      "logps/rejected": -582.2371826171875,
+      "loss": 0.5327,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.5511300563812256,
+      "rewards/margins": 1.97564697265625,
+      "rewards/rejected": -3.5267767906188965,
+      "step": 6720
+    },
+    {
+      "epoch": 0.9907257470925953,
+      "grad_norm": 177.86458994718856,
+      "learning_rate": 1.3102197069067567e-10,
+      "logits/chosen": -1.2806065082550049,
+      "logits/rejected": -0.7323794960975647,
+      "logps/chosen": -455.69659423828125,
+      "logps/rejected": -587.4266357421875,
+      "loss": 0.4243,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.9524253606796265,
+      "rewards/margins": 1.4952733516693115,
+      "rewards/rejected": -3.4476981163024902,
+      "step": 6730
+    },
+    {
+      "epoch": 0.9921978507286913,
+      "grad_norm": 103.03628687622312,
+      "learning_rate": 9.273119485386783e-11,
+      "logits/chosen": -1.3981177806854248,
+      "logits/rejected": -1.2410085201263428,
+      "logps/chosen": -490.24981689453125,
+      "logps/rejected": -645.2311401367188,
+      "loss": 0.5337,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.1277194023132324,
+      "rewards/margins": 1.3783842325210571,
+      "rewards/rejected": -3.506103038787842,
+      "step": 6740
+    },
+    {
+      "epoch": 0.9936699543647873,
+      "grad_norm": 116.5064689963164,
+      "learning_rate": 6.104079820390407e-11,
+      "logits/chosen": -1.4266693592071533,
+      "logits/rejected": -1.2530936002731323,
+      "logps/chosen": -511.64459228515625,
+      "logps/rejected": -556.6649169921875,
+      "loss": 0.5815,
+      "rewards/accuracies": 0.5666666626930237,
+      "rewards/chosen": -2.1102561950683594,
+      "rewards/margins": 0.519591212272644,
+      "rewards/rejected": -2.629847288131714,
+      "step": 6750
+    },
+    {
+      "epoch": 0.9951420580008833,
+      "grad_norm": 97.30029923815557,
+      "learning_rate": 3.595161772582123e-11,
+      "logits/chosen": -1.3649976253509521,
+      "logits/rejected": -1.0474755764007568,
+      "logps/chosen": -344.64178466796875,
+      "logps/rejected": -458.5502014160156,
+      "loss": 0.522,
+      "rewards/accuracies": 0.7666666507720947,
+      "rewards/chosen": -1.2290351390838623,
+      "rewards/margins": 1.540971279144287,
+      "rewards/rejected": -2.7700061798095703,
+      "step": 6760
+    },
+    {
+      "epoch": 0.9966141616369792,
+      "grad_norm": 179.81145239599982,
+      "learning_rate": 1.7464316057785866e-11,
+      "logits/chosen": -1.4237736463546753,
+      "logits/rejected": -1.0620884895324707,
+      "logps/chosen": -453.30889892578125,
+      "logps/rejected": -555.2007446289062,
+      "loss": 0.4954,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.6915595531463623,
+      "rewards/margins": 1.1138288974761963,
+      "rewards/rejected": -2.8053882122039795,
+      "step": 6770
+    },
+    {
+      "epoch": 0.9980862652730752,
+      "grad_norm": 193.13835305388184,
+      "learning_rate": 5.5793814737192805e-12,
+      "logits/chosen": -1.2985987663269043,
+      "logits/rejected": -1.2492955923080444,
+      "logps/chosen": -500.07122802734375,
+      "logps/rejected": -522.789306640625,
+      "loss": 0.5305,
+      "rewards/accuracies": 0.6333333849906921,
+      "rewards/chosen": -2.213953971862793,
+      "rewards/margins": 0.6303444504737854,
+      "rewards/rejected": -2.8442983627319336,
+      "step": 6780
+    },
+    {
+      "epoch": 0.9995583689091712,
+      "grad_norm": 116.06156637115119,
+      "learning_rate": 2.9712787039115617e-13,
+      "logits/chosen": -1.0107321739196777,
+      "logits/rejected": -0.636509120464325,
+      "logps/chosen": -403.2305603027344,
+      "logps/rejected": -516.6555786132812,
+      "loss": 0.5284,
+      "rewards/accuracies": 0.6666666269302368,
+      "rewards/chosen": -1.8709781169891357,
+      "rewards/margins": 1.2671401500701904,
+      "rewards/rejected": -3.138118267059326,
+      "step": 6790
+    },
+    {
+      "epoch": 1.0,
+      "step": 6793,
+      "total_flos": 0.0,
+      "train_loss": 0.44209186683851615,
+      "train_runtime": 28094.4071,
+      "train_samples_per_second": 2.176,
+      "train_steps_per_second": 0.242
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 6793,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}