{ "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 500, "global_step": 6793, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.00014721036360959813, "grad_norm": 29.718448026305357, "learning_rate": 7.352941176470588e-10, "logits/chosen": -3.0151314735412598, "logits/rejected": -2.7100119590759277, "logps/chosen": -416.9676208496094, "logps/rejected": -87.89925384521484, "loss": 0.6931, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1 }, { "epoch": 0.0014721036360959812, "grad_norm": 34.522042629540266, "learning_rate": 7.352941176470588e-09, "logits/chosen": -2.7276546955108643, "logits/rejected": -2.696035623550415, "logps/chosen": -329.0189514160156, "logps/rejected": -228.99288940429688, "loss": 0.6934, "rewards/accuracies": 0.40740740299224854, "rewards/chosen": 8.75515106599778e-05, "rewards/margins": 0.0004485688405111432, "rewards/rejected": -0.0003610174171626568, "step": 10 }, { "epoch": 0.0029442072721919624, "grad_norm": 29.803944573152652, "learning_rate": 1.4705882352941176e-08, "logits/chosen": -2.7379086017608643, "logits/rejected": -2.7686338424682617, "logps/chosen": -257.52764892578125, "logps/rejected": -231.6560516357422, "loss": 0.6932, "rewards/accuracies": 0.4333333969116211, "rewards/chosen": -0.0014953905483707786, "rewards/margins": -0.0011777242179960012, "rewards/rejected": -0.00031766638858243823, "step": 20 }, { "epoch": 0.004416310908287944, "grad_norm": 27.15125265339492, "learning_rate": 2.2058823529411764e-08, "logits/chosen": -2.8205478191375732, "logits/rejected": -2.8088126182556152, "logps/chosen": -202.71090698242188, "logps/rejected": -208.66604614257812, "loss": 0.6935, "rewards/accuracies": 0.36666664481163025, "rewards/chosen": -0.000266118033323437, "rewards/margins": -0.0014482138212770224, "rewards/rejected": 0.0011820958461612463, "step": 30 }, { "epoch": 0.005888414544383925, "grad_norm": 27.748767219344842, "learning_rate": 2.941176470588235e-08, "logits/chosen": -2.7897677421569824, "logits/rejected": -2.7368621826171875, "logps/chosen": -244.4735870361328, "logps/rejected": -392.8205871582031, "loss": 0.6939, "rewards/accuracies": 0.3333333134651184, "rewards/chosen": -0.00016161819803528488, "rewards/margins": -0.0008207018254324794, "rewards/rejected": 0.0006590837729163468, "step": 40 }, { "epoch": 0.007360518180479906, "grad_norm": 38.32874910808029, "learning_rate": 3.676470588235294e-08, "logits/chosen": -2.756470203399658, "logits/rejected": -2.7820448875427246, "logps/chosen": -273.6413269042969, "logps/rejected": -217.2002716064453, "loss": 0.6933, "rewards/accuracies": 0.4333333373069763, "rewards/chosen": 0.0007894287118688226, "rewards/margins": 0.0006995893782004714, "rewards/rejected": 8.983933366835117e-05, "step": 50 }, { "epoch": 0.008832621816575887, "grad_norm": 34.64234916556576, "learning_rate": 4.411764705882353e-08, "logits/chosen": -2.8398597240448, "logits/rejected": -2.82981014251709, "logps/chosen": -320.47674560546875, "logps/rejected": -330.9267883300781, "loss": 0.6929, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": 0.002479543210938573, "rewards/margins": 0.001817501150071621, "rewards/rejected": 0.0006620419444516301, "step": 60 }, { "epoch": 0.010304725452671868, "grad_norm": 30.591214827639572, "learning_rate": 5.147058823529411e-08, "logits/chosen": -2.656278133392334, "logits/rejected": -2.6301004886627197, "logps/chosen": -247.33352661132812, "logps/rejected": -297.91912841796875, "loss": 0.6932, "rewards/accuracies": 0.5333333015441895, "rewards/chosen": 0.0015019330894574523, "rewards/margins": 0.0005758283077739179, "rewards/rejected": 0.0009261048398911953, "step": 70 }, { "epoch": 0.01177682908876785, "grad_norm": 29.311732036148747, "learning_rate": 5.88235294117647e-08, "logits/chosen": -2.886997938156128, "logits/rejected": -2.8162360191345215, "logps/chosen": -254.671630859375, "logps/rejected": -228.0240020751953, "loss": 0.6925, "rewards/accuracies": 0.4000000059604645, "rewards/chosen": 0.0005007624276913702, "rewards/margins": -0.0004886452225036919, "rewards/rejected": 0.0009894075337797403, "step": 80 }, { "epoch": 0.01324893272486383, "grad_norm": 28.19375151099229, "learning_rate": 6.617647058823529e-08, "logits/chosen": -2.890990734100342, "logits/rejected": -2.8566465377807617, "logps/chosen": -297.2953186035156, "logps/rejected": -230.1473388671875, "loss": 0.6925, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": 0.0029033743776381016, "rewards/margins": 0.0014815034810453653, "rewards/rejected": 0.001421870430931449, "step": 90 }, { "epoch": 0.014721036360959812, "grad_norm": 29.517686848229378, "learning_rate": 7.352941176470588e-08, "logits/chosen": -2.8629086017608643, "logits/rejected": -2.7863824367523193, "logps/chosen": -256.60455322265625, "logps/rejected": -211.39645385742188, "loss": 0.6923, "rewards/accuracies": 0.43333330750465393, "rewards/chosen": 0.0015266563277691603, "rewards/margins": 0.0008079732651822269, "rewards/rejected": 0.0007186831790022552, "step": 100 }, { "epoch": 0.016193139997055794, "grad_norm": 28.80409325053622, "learning_rate": 8.088235294117647e-08, "logits/chosen": -2.901120901107788, "logits/rejected": -2.8686041831970215, "logps/chosen": -357.46038818359375, "logps/rejected": -284.4308166503906, "loss": 0.6914, "rewards/accuracies": 0.5, "rewards/chosen": 0.007152503821998835, "rewards/margins": 0.002613933989778161, "rewards/rejected": 0.00453856959939003, "step": 110 }, { "epoch": 0.017665243633151775, "grad_norm": 36.06810060733407, "learning_rate": 8.823529411764706e-08, "logits/chosen": -2.766767740249634, "logits/rejected": -2.7674272060394287, "logps/chosen": -250.0283660888672, "logps/rejected": -270.9175720214844, "loss": 0.6924, "rewards/accuracies": 0.46666663885116577, "rewards/chosen": 0.0038150239270180464, "rewards/margins": 0.00018358106899540871, "rewards/rejected": 0.003631443250924349, "step": 120 }, { "epoch": 0.019137347269247755, "grad_norm": 32.75221872729468, "learning_rate": 9.558823529411763e-08, "logits/chosen": -2.8407626152038574, "logits/rejected": -2.8098511695861816, "logps/chosen": -284.9811096191406, "logps/rejected": -217.84854125976562, "loss": 0.6909, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": 0.009724515490233898, "rewards/margins": 0.006964170839637518, "rewards/rejected": 0.0027603451162576675, "step": 130 }, { "epoch": 0.020609450905343735, "grad_norm": 32.40114017346329, "learning_rate": 1.0294117647058822e-07, "logits/chosen": -2.87676739692688, "logits/rejected": -2.8542282581329346, "logps/chosen": -194.10653686523438, "logps/rejected": -192.06259155273438, "loss": 0.6911, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": 0.011662951670587063, "rewards/margins": 0.006347318179905415, "rewards/rejected": 0.005315634422004223, "step": 140 }, { "epoch": 0.022081554541439716, "grad_norm": 32.6183972925703, "learning_rate": 1.1029411764705881e-07, "logits/chosen": -2.836061954498291, "logits/rejected": -2.7306556701660156, "logps/chosen": -262.98590087890625, "logps/rejected": -223.1548309326172, "loss": 0.6901, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": 0.01592249795794487, "rewards/margins": 0.0072413100861012936, "rewards/rejected": 0.008681186474859715, "step": 150 }, { "epoch": 0.0235536581775357, "grad_norm": 26.829953137326793, "learning_rate": 1.176470588235294e-07, "logits/chosen": -2.8711628913879395, "logits/rejected": -2.7835397720336914, "logps/chosen": -224.1538543701172, "logps/rejected": -213.1654815673828, "loss": 0.6887, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": 0.014381324872374535, "rewards/margins": 0.0020202070008963346, "rewards/rejected": 0.012361117638647556, "step": 160 }, { "epoch": 0.02502576181363168, "grad_norm": 43.31476786218589, "learning_rate": 1.25e-07, "logits/chosen": -2.8009700775146484, "logits/rejected": -2.8119311332702637, "logps/chosen": -282.51092529296875, "logps/rejected": -246.3946990966797, "loss": 0.6871, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": 0.03146267682313919, "rewards/margins": 0.016998907551169395, "rewards/rejected": 0.01446377020329237, "step": 170 }, { "epoch": 0.02649786544972766, "grad_norm": 32.70338126838971, "learning_rate": 1.3235294117647057e-07, "logits/chosen": -2.873365640640259, "logits/rejected": -2.8650002479553223, "logps/chosen": -198.95913696289062, "logps/rejected": -216.2865447998047, "loss": 0.6863, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": 0.029899191111326218, "rewards/margins": 0.012989306822419167, "rewards/rejected": 0.016909882426261902, "step": 180 }, { "epoch": 0.02796996908582364, "grad_norm": 27.647809254073966, "learning_rate": 1.3970588235294117e-07, "logits/chosen": -2.8443760871887207, "logits/rejected": -2.8531670570373535, "logps/chosen": -238.43276977539062, "logps/rejected": -254.6396026611328, "loss": 0.6839, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": 0.03018432855606079, "rewards/margins": 0.011908235028386116, "rewards/rejected": 0.018276093527674675, "step": 190 }, { "epoch": 0.029442072721919624, "grad_norm": 31.006923353950068, "learning_rate": 1.4705882352941175e-07, "logits/chosen": -2.8793869018554688, "logits/rejected": -2.809866428375244, "logps/chosen": -319.7313537597656, "logps/rejected": -248.39608764648438, "loss": 0.6827, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.04280875250697136, "rewards/margins": 0.02830171212553978, "rewards/rejected": 0.014507037587463856, "step": 200 }, { "epoch": 0.030914176358015605, "grad_norm": 35.15155577786183, "learning_rate": 1.5441176470588236e-07, "logits/chosen": -2.7579071521759033, "logits/rejected": -2.750349760055542, "logps/chosen": -250.6804962158203, "logps/rejected": -204.61766052246094, "loss": 0.676, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": 0.043508581817150116, "rewards/margins": 0.02954995632171631, "rewards/rejected": 0.013958625495433807, "step": 210 }, { "epoch": 0.03238627999411159, "grad_norm": 41.541119045331676, "learning_rate": 1.6176470588235293e-07, "logits/chosen": -2.755091667175293, "logits/rejected": -2.730134963989258, "logps/chosen": -241.91476440429688, "logps/rejected": -188.8052520751953, "loss": 0.6712, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.041111722588539124, "rewards/margins": 0.034332290291786194, "rewards/rejected": 0.006779427640140057, "step": 220 }, { "epoch": 0.033858383630207566, "grad_norm": 35.780007512350444, "learning_rate": 1.6911764705882354e-07, "logits/chosen": -2.8124828338623047, "logits/rejected": -2.7572226524353027, "logps/chosen": -346.2286682128906, "logps/rejected": -255.1610107421875, "loss": 0.6761, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": 0.0636453926563263, "rewards/margins": 0.04507563263177872, "rewards/rejected": 0.018569760024547577, "step": 230 }, { "epoch": 0.03533048726630355, "grad_norm": 24.867225994165324, "learning_rate": 1.764705882352941e-07, "logits/chosen": -2.8328232765197754, "logits/rejected": -2.764782428741455, "logps/chosen": -259.3244934082031, "logps/rejected": -253.60427856445312, "loss": 0.675, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.03747386485338211, "rewards/margins": 0.03954165801405907, "rewards/rejected": -0.0020677968859672546, "step": 240 }, { "epoch": 0.036802590902399526, "grad_norm": 31.837234874996586, "learning_rate": 1.8382352941176472e-07, "logits/chosen": -2.6840827465057373, "logits/rejected": -2.723935842514038, "logps/chosen": -238.89794921875, "logps/rejected": -240.5414581298828, "loss": 0.6754, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": 0.032973356544971466, "rewards/margins": 0.048632390797138214, "rewards/rejected": -0.015659034252166748, "step": 250 }, { "epoch": 0.03827469453849551, "grad_norm": 29.646802079354163, "learning_rate": 1.9117647058823527e-07, "logits/chosen": -2.88181734085083, "logits/rejected": -2.779343605041504, "logps/chosen": -266.6003723144531, "logps/rejected": -222.57400512695312, "loss": 0.673, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -0.003907797392457724, "rewards/margins": 0.012058199383318424, "rewards/rejected": -0.01596599444746971, "step": 260 }, { "epoch": 0.039746798174591494, "grad_norm": 26.851190889186874, "learning_rate": 1.9852941176470587e-07, "logits/chosen": -2.799879789352417, "logits/rejected": -2.848215341567993, "logps/chosen": -292.47210693359375, "logps/rejected": -280.51190185546875, "loss": 0.6589, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.027005229145288467, "rewards/margins": 0.039683397859334946, "rewards/rejected": -0.012678168714046478, "step": 270 }, { "epoch": 0.04121890181068747, "grad_norm": 30.64659123108073, "learning_rate": 2.0588235294117645e-07, "logits/chosen": -2.840538263320923, "logits/rejected": -2.781792640686035, "logps/chosen": -315.31658935546875, "logps/rejected": -288.5542297363281, "loss": 0.6538, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.05827077478170395, "rewards/margins": 0.07762565463781357, "rewards/rejected": -0.01935487426817417, "step": 280 }, { "epoch": 0.042691005446783455, "grad_norm": 38.622609391819914, "learning_rate": 2.1323529411764705e-07, "logits/chosen": -2.829793930053711, "logits/rejected": -2.8271484375, "logps/chosen": -268.45330810546875, "logps/rejected": -226.24526977539062, "loss": 0.6623, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": 0.04187547042965889, "rewards/margins": 0.09543410688638687, "rewards/rejected": -0.05355863645672798, "step": 290 }, { "epoch": 0.04416310908287943, "grad_norm": 32.02994765741851, "learning_rate": 2.2058823529411763e-07, "logits/chosen": -2.820500373840332, "logits/rejected": -2.7968363761901855, "logps/chosen": -271.98992919921875, "logps/rejected": -247.7967987060547, "loss": 0.658, "rewards/accuracies": 0.7000000476837158, "rewards/chosen": -0.03903895989060402, "rewards/margins": 0.0660947933793068, "rewards/rejected": -0.10513375699520111, "step": 300 }, { "epoch": 0.045635212718975415, "grad_norm": 45.6308084481084, "learning_rate": 2.2794117647058823e-07, "logits/chosen": -2.9065310955047607, "logits/rejected": -2.8418185710906982, "logps/chosen": -309.17889404296875, "logps/rejected": -269.48394775390625, "loss": 0.6641, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.003005195641890168, "rewards/margins": 0.10960595309734344, "rewards/rejected": -0.10660076141357422, "step": 310 }, { "epoch": 0.0471073163550714, "grad_norm": 38.680300557067135, "learning_rate": 2.352941176470588e-07, "logits/chosen": -2.7667911052703857, "logits/rejected": -2.745961904525757, "logps/chosen": -259.071044921875, "logps/rejected": -268.68756103515625, "loss": 0.6419, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -0.04463445395231247, "rewards/margins": 0.1286865472793579, "rewards/rejected": -0.1733209788799286, "step": 320 }, { "epoch": 0.048579419991167376, "grad_norm": 43.0929608927417, "learning_rate": 2.426470588235294e-07, "logits/chosen": -2.805534839630127, "logits/rejected": -2.7685582637786865, "logps/chosen": -234.90036010742188, "logps/rejected": -213.7690887451172, "loss": 0.6416, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -0.04509614408016205, "rewards/margins": 0.13314001262187958, "rewards/rejected": -0.17823615670204163, "step": 330 }, { "epoch": 0.05005152362726336, "grad_norm": 32.26871001944957, "learning_rate": 2.5e-07, "logits/chosen": -2.8668112754821777, "logits/rejected": -2.820902109146118, "logps/chosen": -304.05535888671875, "logps/rejected": -353.029296875, "loss": 0.6517, "rewards/accuracies": 0.5333333611488342, "rewards/chosen": -0.08451329171657562, "rewards/margins": 0.08599463105201721, "rewards/rejected": -0.17050793766975403, "step": 340 }, { "epoch": 0.051523627263359344, "grad_norm": 44.69532980871668, "learning_rate": 2.5735294117647057e-07, "logits/chosen": -2.7562334537506104, "logits/rejected": -2.730844020843506, "logps/chosen": -316.5646057128906, "logps/rejected": -269.04498291015625, "loss": 0.6456, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -0.11849607527256012, "rewards/margins": 0.12490792572498322, "rewards/rejected": -0.24340395629405975, "step": 350 }, { "epoch": 0.05299573089945532, "grad_norm": 34.892260900939654, "learning_rate": 2.6470588235294114e-07, "logits/chosen": -2.8204593658447266, "logits/rejected": -2.8178982734680176, "logps/chosen": -250.16171264648438, "logps/rejected": -244.96011352539062, "loss": 0.6529, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -0.14288079738616943, "rewards/margins": 0.11003166437149048, "rewards/rejected": -0.2529124915599823, "step": 360 }, { "epoch": 0.054467834535551304, "grad_norm": 40.19716854439912, "learning_rate": 2.720588235294117e-07, "logits/chosen": -2.8387558460235596, "logits/rejected": -2.8843367099761963, "logps/chosen": -269.55828857421875, "logps/rejected": -285.814453125, "loss": 0.597, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -0.0735350102186203, "rewards/margins": 0.20208704471588135, "rewards/rejected": -0.27562204003334045, "step": 370 }, { "epoch": 0.05593993817164728, "grad_norm": 40.430296299126645, "learning_rate": 2.7941176470588235e-07, "logits/chosen": -2.9129586219787598, "logits/rejected": -2.863222599029541, "logps/chosen": -300.028076171875, "logps/rejected": -262.0947265625, "loss": 0.6348, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -0.3237457573413849, "rewards/margins": 0.13909229636192322, "rewards/rejected": -0.4628380835056305, "step": 380 }, { "epoch": 0.057412041807743265, "grad_norm": 38.1179702652076, "learning_rate": 2.8676470588235293e-07, "logits/chosen": -2.856393337249756, "logits/rejected": -2.845912456512451, "logps/chosen": -304.79083251953125, "logps/rejected": -298.27838134765625, "loss": 0.6694, "rewards/accuracies": 0.5333333611488342, "rewards/chosen": -0.44157400727272034, "rewards/margins": -0.01729520596563816, "rewards/rejected": -0.42427879571914673, "step": 390 }, { "epoch": 0.05888414544383925, "grad_norm": 37.66484657174034, "learning_rate": 2.941176470588235e-07, "logits/chosen": -2.846362590789795, "logits/rejected": -2.7886412143707275, "logps/chosen": -251.76260375976562, "logps/rejected": -277.6621398925781, "loss": 0.6068, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -0.39646148681640625, "rewards/margins": 0.20760174095630646, "rewards/rejected": -0.6040632128715515, "step": 400 }, { "epoch": 0.060356249079935226, "grad_norm": 44.823295544217466, "learning_rate": 3.014705882352941e-07, "logits/chosen": -2.7775685787200928, "logits/rejected": -2.7247066497802734, "logps/chosen": -342.48760986328125, "logps/rejected": -346.5843200683594, "loss": 0.6163, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -0.36908066272735596, "rewards/margins": 0.1995377540588379, "rewards/rejected": -0.5686183571815491, "step": 410 }, { "epoch": 0.06182835271603121, "grad_norm": 57.91546494547815, "learning_rate": 3.088235294117647e-07, "logits/chosen": -2.9392282962799072, "logits/rejected": -2.874932050704956, "logps/chosen": -355.48284912109375, "logps/rejected": -334.55877685546875, "loss": 0.6611, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -0.35706859827041626, "rewards/margins": 0.08930377662181854, "rewards/rejected": -0.4463724195957184, "step": 420 }, { "epoch": 0.0633004563521272, "grad_norm": 40.864948374445554, "learning_rate": 3.161764705882353e-07, "logits/chosen": -2.813310146331787, "logits/rejected": -2.8060102462768555, "logps/chosen": -189.114013671875, "logps/rejected": -223.2728729248047, "loss": 0.6161, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -0.1683691293001175, "rewards/margins": 0.22235913574695587, "rewards/rejected": -0.39072829484939575, "step": 430 }, { "epoch": 0.06477255998822318, "grad_norm": 39.57400537972088, "learning_rate": 3.2352941176470586e-07, "logits/chosen": -2.841413974761963, "logits/rejected": -2.83341646194458, "logps/chosen": -226.3544921875, "logps/rejected": -234.64328002929688, "loss": 0.595, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -0.12442722171545029, "rewards/margins": 0.3463088274002075, "rewards/rejected": -0.470736026763916, "step": 440 }, { "epoch": 0.06624466362431915, "grad_norm": 40.15876245255608, "learning_rate": 3.3088235294117644e-07, "logits/chosen": -2.8721110820770264, "logits/rejected": -2.8159689903259277, "logps/chosen": -383.03045654296875, "logps/rejected": -354.8152160644531, "loss": 0.617, "rewards/accuracies": 0.5333333611488342, "rewards/chosen": -0.5107864141464233, "rewards/margins": 0.07485867291688919, "rewards/rejected": -0.5856450796127319, "step": 450 }, { "epoch": 0.06771676726041513, "grad_norm": 50.420023450070595, "learning_rate": 3.3823529411764707e-07, "logits/chosen": -2.9715359210968018, "logits/rejected": -2.8942055702209473, "logps/chosen": -316.4211730957031, "logps/rejected": -294.3646240234375, "loss": 0.6338, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -0.3568728566169739, "rewards/margins": 0.4465089738368988, "rewards/rejected": -0.8033817410469055, "step": 460 }, { "epoch": 0.06918887089651111, "grad_norm": 45.45079803725552, "learning_rate": 3.4558823529411765e-07, "logits/chosen": -2.894871950149536, "logits/rejected": -2.882598638534546, "logps/chosen": -268.9466247558594, "logps/rejected": -284.09405517578125, "loss": 0.5799, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -0.5613805651664734, "rewards/margins": 0.30428066849708557, "rewards/rejected": -0.8656612634658813, "step": 470 }, { "epoch": 0.0706609745326071, "grad_norm": 39.98380606716504, "learning_rate": 3.529411764705882e-07, "logits/chosen": -2.808845281600952, "logits/rejected": -2.765450954437256, "logps/chosen": -341.7030334472656, "logps/rejected": -362.5021667480469, "loss": 0.5428, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -0.6110513806343079, "rewards/margins": 0.6107190847396851, "rewards/rejected": -1.2217704057693481, "step": 480 }, { "epoch": 0.07213307816870308, "grad_norm": 52.16434389424912, "learning_rate": 3.602941176470588e-07, "logits/chosen": -2.913093090057373, "logits/rejected": -2.8623852729797363, "logps/chosen": -376.265625, "logps/rejected": -357.77996826171875, "loss": 0.6358, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -0.6043509244918823, "rewards/margins": 0.2012513428926468, "rewards/rejected": -0.8056022524833679, "step": 490 }, { "epoch": 0.07360518180479905, "grad_norm": 49.42423787089578, "learning_rate": 3.6764705882352943e-07, "logits/chosen": -2.8766541481018066, "logits/rejected": -2.901610851287842, "logps/chosen": -248.31686401367188, "logps/rejected": -313.4378967285156, "loss": 0.5365, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -0.4482509195804596, "rewards/margins": 0.4299069344997406, "rewards/rejected": -0.8781577944755554, "step": 500 }, { "epoch": 0.07507728544089504, "grad_norm": 32.57183193077679, "learning_rate": 3.75e-07, "logits/chosen": -2.821455717086792, "logits/rejected": -2.8248238563537598, "logps/chosen": -348.87811279296875, "logps/rejected": -407.82110595703125, "loss": 0.5858, "rewards/accuracies": 0.76666659116745, "rewards/chosen": -0.3897936940193176, "rewards/margins": 0.7289079427719116, "rewards/rejected": -1.1187015771865845, "step": 510 }, { "epoch": 0.07654938907699102, "grad_norm": 70.79198965743817, "learning_rate": 3.8235294117647053e-07, "logits/chosen": -2.70387601852417, "logits/rejected": -2.607253074645996, "logps/chosen": -290.9710388183594, "logps/rejected": -309.3666687011719, "loss": 0.6392, "rewards/accuracies": 0.6333333849906921, "rewards/chosen": -0.7217598557472229, "rewards/margins": 0.16696974635124207, "rewards/rejected": -0.8887295722961426, "step": 520 }, { "epoch": 0.078021492713087, "grad_norm": 68.7838639999348, "learning_rate": 3.8970588235294116e-07, "logits/chosen": -2.878333330154419, "logits/rejected": -2.8191580772399902, "logps/chosen": -337.52203369140625, "logps/rejected": -348.9778747558594, "loss": 0.6487, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -0.6876872181892395, "rewards/margins": 0.25353384017944336, "rewards/rejected": -0.9412209391593933, "step": 530 }, { "epoch": 0.07949359634918299, "grad_norm": 72.45119032304852, "learning_rate": 3.9705882352941174e-07, "logits/chosen": -3.028770923614502, "logits/rejected": -2.9213104248046875, "logps/chosen": -358.0929260253906, "logps/rejected": -314.4250183105469, "loss": 0.6274, "rewards/accuracies": 0.5333333611488342, "rewards/chosen": -0.4906277060508728, "rewards/margins": 0.09770762920379639, "rewards/rejected": -0.588335394859314, "step": 540 }, { "epoch": 0.08096569998527896, "grad_norm": 42.155038860783506, "learning_rate": 4.044117647058823e-07, "logits/chosen": -2.8411173820495605, "logits/rejected": -2.8484601974487305, "logps/chosen": -366.4306335449219, "logps/rejected": -394.5599060058594, "loss": 0.509, "rewards/accuracies": 0.76666659116745, "rewards/chosen": -0.3539455533027649, "rewards/margins": 0.6814759373664856, "rewards/rejected": -1.0354214906692505, "step": 550 }, { "epoch": 0.08243780362137494, "grad_norm": 59.00223055195894, "learning_rate": 4.117647058823529e-07, "logits/chosen": -2.9224987030029297, "logits/rejected": -2.854747772216797, "logps/chosen": -316.6570739746094, "logps/rejected": -348.1865234375, "loss": 0.5664, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -0.7669118642807007, "rewards/margins": 0.4014472961425781, "rewards/rejected": -1.1683591604232788, "step": 560 }, { "epoch": 0.08390990725747093, "grad_norm": 158.17054517954566, "learning_rate": 4.191176470588235e-07, "logits/chosen": -2.860236883163452, "logits/rejected": -2.782071590423584, "logps/chosen": -366.7201232910156, "logps/rejected": -369.0834655761719, "loss": 0.6191, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -0.7171522974967957, "rewards/margins": 0.708254873752594, "rewards/rejected": -1.4254071712493896, "step": 570 }, { "epoch": 0.08538201089356691, "grad_norm": 100.00875893504237, "learning_rate": 4.264705882352941e-07, "logits/chosen": -2.7357230186462402, "logits/rejected": -2.732067584991455, "logps/chosen": -403.13421630859375, "logps/rejected": -354.78961181640625, "loss": 0.5898, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.0905609130859375, "rewards/margins": 0.23605385422706604, "rewards/rejected": -1.3266146183013916, "step": 580 }, { "epoch": 0.08685411452966289, "grad_norm": 105.63343146580014, "learning_rate": 4.338235294117647e-07, "logits/chosen": -2.7384467124938965, "logits/rejected": -2.7308027744293213, "logps/chosen": -354.20928955078125, "logps/rejected": -388.06048583984375, "loss": 0.5831, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -0.7502145171165466, "rewards/margins": 0.5446814298629761, "rewards/rejected": -1.294895887374878, "step": 590 }, { "epoch": 0.08832621816575886, "grad_norm": 114.38595516000808, "learning_rate": 4.4117647058823526e-07, "logits/chosen": -2.5501580238342285, "logits/rejected": -2.652919054031372, "logps/chosen": -336.08380126953125, "logps/rejected": -418.5784606933594, "loss": 0.5505, "rewards/accuracies": 0.73333340883255, "rewards/chosen": -0.7092531323432922, "rewards/margins": 0.719337522983551, "rewards/rejected": -1.4285906553268433, "step": 600 }, { "epoch": 0.08979832180185485, "grad_norm": 141.59819490328758, "learning_rate": 4.485294117647059e-07, "logits/chosen": -2.716247081756592, "logits/rejected": -2.6971218585968018, "logps/chosen": -330.3377990722656, "logps/rejected": -390.364013671875, "loss": 0.5446, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -0.7154955863952637, "rewards/margins": 0.6936029195785522, "rewards/rejected": -1.4090986251831055, "step": 610 }, { "epoch": 0.09127042543795083, "grad_norm": 46.65751534451082, "learning_rate": 4.5588235294117646e-07, "logits/chosen": -2.858609914779663, "logits/rejected": -2.794933557510376, "logps/chosen": -318.4283752441406, "logps/rejected": -318.87432861328125, "loss": 0.5765, "rewards/accuracies": 0.5999999642372131, "rewards/chosen": -0.773520290851593, "rewards/margins": 0.44950708746910095, "rewards/rejected": -1.2230274677276611, "step": 620 }, { "epoch": 0.09274252907404681, "grad_norm": 63.97315843764817, "learning_rate": 4.6323529411764704e-07, "logits/chosen": -2.817518711090088, "logits/rejected": -2.7873594760894775, "logps/chosen": -395.9840087890625, "logps/rejected": -406.97479248046875, "loss": 0.4769, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -0.8255616426467896, "rewards/margins": 0.7154530882835388, "rewards/rejected": -1.5410146713256836, "step": 630 }, { "epoch": 0.0942146327101428, "grad_norm": 281.24021060780353, "learning_rate": 4.705882352941176e-07, "logits/chosen": -2.747581958770752, "logits/rejected": -2.699913501739502, "logps/chosen": -392.9543762207031, "logps/rejected": -419.4530334472656, "loss": 0.6503, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -1.1352064609527588, "rewards/margins": 0.42551788687705994, "rewards/rejected": -1.5607244968414307, "step": 640 }, { "epoch": 0.09568673634623878, "grad_norm": 60.53577553719157, "learning_rate": 4.779411764705882e-07, "logits/chosen": -2.7471470832824707, "logits/rejected": -2.6654038429260254, "logps/chosen": -347.45501708984375, "logps/rejected": -396.5620422363281, "loss": 0.5248, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -0.5082324147224426, "rewards/margins": 0.6192538738250732, "rewards/rejected": -1.1274863481521606, "step": 650 }, { "epoch": 0.09715883998233475, "grad_norm": 82.56737879385132, "learning_rate": 4.852941176470588e-07, "logits/chosen": -2.751802444458008, "logits/rejected": -2.6738271713256836, "logps/chosen": -337.10150146484375, "logps/rejected": -350.31634521484375, "loss": 0.575, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -0.8105621337890625, "rewards/margins": 0.44342517852783203, "rewards/rejected": -1.2539875507354736, "step": 660 }, { "epoch": 0.09863094361843074, "grad_norm": 126.84675528859458, "learning_rate": 4.926470588235295e-07, "logits/chosen": -2.6607487201690674, "logits/rejected": -2.690328359603882, "logps/chosen": -375.0699157714844, "logps/rejected": -356.2629699707031, "loss": 0.5818, "rewards/accuracies": 0.5666667222976685, "rewards/chosen": -0.9144572019577026, "rewards/margins": 0.3163110613822937, "rewards/rejected": -1.2307683229446411, "step": 670 }, { "epoch": 0.10010304725452672, "grad_norm": 88.01654938748439, "learning_rate": 5e-07, "logits/chosen": -2.810364246368408, "logits/rejected": -2.6994426250457764, "logps/chosen": -327.703369140625, "logps/rejected": -312.79620361328125, "loss": 0.6337, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -0.830489993095398, "rewards/margins": 0.4034651815891266, "rewards/rejected": -1.2339551448822021, "step": 680 }, { "epoch": 0.1015751508906227, "grad_norm": 67.9023375850433, "learning_rate": 4.999966985858302e-07, "logits/chosen": -2.7782535552978516, "logits/rejected": -2.7502403259277344, "logps/chosen": -320.7219543457031, "logps/rejected": -359.96759033203125, "loss": 0.5899, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -0.7388945817947388, "rewards/margins": 0.2782880365848541, "rewards/rejected": -1.0171825885772705, "step": 690 }, { "epoch": 0.10304725452671869, "grad_norm": 63.88394385411192, "learning_rate": 4.999867944305156e-07, "logits/chosen": -2.7173736095428467, "logits/rejected": -2.7157700061798096, "logps/chosen": -260.1662902832031, "logps/rejected": -324.9969787597656, "loss": 0.5417, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -0.646327793598175, "rewards/margins": 0.5964962244033813, "rewards/rejected": -1.2428240776062012, "step": 700 }, { "epoch": 0.10451935816281466, "grad_norm": 82.12543215083453, "learning_rate": 4.99970287795638e-07, "logits/chosen": -2.7450194358825684, "logits/rejected": -2.6951887607574463, "logps/chosen": -377.6488037109375, "logps/rejected": -471.7896423339844, "loss": 0.6345, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -1.2469216585159302, "rewards/margins": 0.43245401978492737, "rewards/rejected": -1.6793756484985352, "step": 710 }, { "epoch": 0.10599146179891064, "grad_norm": 76.26325826475778, "learning_rate": 4.999471791171592e-07, "logits/chosen": -2.6871371269226074, "logits/rejected": -2.713042736053467, "logps/chosen": -373.04498291015625, "logps/rejected": -366.93609619140625, "loss": 0.5365, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.154412031173706, "rewards/margins": 0.4296625256538391, "rewards/rejected": -1.58407461643219, "step": 720 }, { "epoch": 0.10746356543500662, "grad_norm": 72.59515441979644, "learning_rate": 4.999174690054098e-07, "logits/chosen": -2.7032949924468994, "logits/rejected": -2.706186532974243, "logps/chosen": -378.016357421875, "logps/rejected": -493.35150146484375, "loss": 0.6098, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.5206677913665771, "rewards/margins": 0.39631330966949463, "rewards/rejected": -1.9169813394546509, "step": 730 }, { "epoch": 0.10893566907110261, "grad_norm": 77.08798499260584, "learning_rate": 4.998811582450728e-07, "logits/chosen": -2.7526307106018066, "logits/rejected": -2.770782947540283, "logps/chosen": -417.216064453125, "logps/rejected": -473.040771484375, "loss": 0.5572, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.1277623176574707, "rewards/margins": 0.6581010818481445, "rewards/rejected": -1.7858635187149048, "step": 740 }, { "epoch": 0.11040777270719859, "grad_norm": 92.20853458400707, "learning_rate": 4.998382477951632e-07, "logits/chosen": -2.7655673027038574, "logits/rejected": -2.77188777923584, "logps/chosen": -330.7748107910156, "logps/rejected": -394.17230224609375, "loss": 0.6361, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.120185375213623, "rewards/margins": 0.358142226934433, "rewards/rejected": -1.4783275127410889, "step": 750 }, { "epoch": 0.11187987634329456, "grad_norm": 64.87979385908508, "learning_rate": 4.997887387890022e-07, "logits/chosen": -2.694700241088867, "logits/rejected": -2.7389893531799316, "logps/chosen": -423.99481201171875, "logps/rejected": -472.5301818847656, "loss": 0.4996, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -0.7210303544998169, "rewards/margins": 0.6826065182685852, "rewards/rejected": -1.4036369323730469, "step": 760 }, { "epoch": 0.11335197997939055, "grad_norm": 62.14673998391493, "learning_rate": 4.997326325341876e-07, "logits/chosen": -2.865445852279663, "logits/rejected": -2.785827875137329, "logps/chosen": -351.143798828125, "logps/rejected": -376.88116455078125, "loss": 0.6837, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.141156554222107, "rewards/margins": 0.18905548751354218, "rewards/rejected": -1.3302119970321655, "step": 770 }, { "epoch": 0.11482408361548653, "grad_norm": 50.25048989730604, "learning_rate": 4.996699305125597e-07, "logits/chosen": -2.6305692195892334, "logits/rejected": -2.6139140129089355, "logps/chosen": -318.23907470703125, "logps/rejected": -363.19854736328125, "loss": 0.6313, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -0.9152249097824097, "rewards/margins": 0.6227025389671326, "rewards/rejected": -1.537927508354187, "step": 780 }, { "epoch": 0.11629618725158251, "grad_norm": 60.098691114202936, "learning_rate": 4.996006343801608e-07, "logits/chosen": -2.580573320388794, "logits/rejected": -2.538261651992798, "logps/chosen": -304.2982482910156, "logps/rejected": -375.99017333984375, "loss": 0.5741, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -0.7302695512771606, "rewards/margins": 0.7375911474227905, "rewards/rejected": -1.4678608179092407, "step": 790 }, { "epoch": 0.1177682908876785, "grad_norm": 94.19572824034412, "learning_rate": 4.99524745967193e-07, "logits/chosen": -2.521172046661377, "logits/rejected": -2.49949312210083, "logps/chosen": -468.8832092285156, "logps/rejected": -470.048583984375, "loss": 0.5629, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.1481328010559082, "rewards/margins": 0.7506190538406372, "rewards/rejected": -1.8987518548965454, "step": 800 }, { "epoch": 0.11924039452377447, "grad_norm": 77.02767987142049, "learning_rate": 4.994422672779687e-07, "logits/chosen": -2.4656782150268555, "logits/rejected": -2.4513707160949707, "logps/chosen": -349.1130065917969, "logps/rejected": -448.8529357910156, "loss": 0.5245, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.2570167779922485, "rewards/margins": 0.6979629993438721, "rewards/rejected": -1.954979658126831, "step": 810 }, { "epoch": 0.12071249815987045, "grad_norm": 107.62300360051363, "learning_rate": 4.993532004908588e-07, "logits/chosen": -2.4160711765289307, "logits/rejected": -2.288529872894287, "logps/chosen": -371.8260803222656, "logps/rejected": -403.8309020996094, "loss": 0.604, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.372617244720459, "rewards/margins": 0.3440636694431305, "rewards/rejected": -1.716680884361267, "step": 820 }, { "epoch": 0.12218460179596644, "grad_norm": 90.2537296603263, "learning_rate": 4.992575479582337e-07, "logits/chosen": -2.3595759868621826, "logits/rejected": -2.309431791305542, "logps/chosen": -489.00909423828125, "logps/rejected": -533.1735229492188, "loss": 0.5051, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.147862195968628, "rewards/margins": 0.6809053421020508, "rewards/rejected": -1.8287674188613892, "step": 830 }, { "epoch": 0.12365670543206242, "grad_norm": 119.79529421225497, "learning_rate": 4.991553122064028e-07, "logits/chosen": -2.269420623779297, "logits/rejected": -2.1465039253234863, "logps/chosen": -460.19476318359375, "logps/rejected": -437.865966796875, "loss": 0.6107, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.6137462854385376, "rewards/margins": 0.4687502980232239, "rewards/rejected": -2.082496404647827, "step": 840 }, { "epoch": 0.1251288090681584, "grad_norm": 75.9177588654981, "learning_rate": 4.990464959355464e-07, "logits/chosen": -2.3251051902770996, "logits/rejected": -2.373628616333008, "logps/chosen": -393.3048095703125, "logps/rejected": -467.0416564941406, "loss": 0.6201, "rewards/accuracies": 0.5666666626930237, "rewards/chosen": -1.5888453722000122, "rewards/margins": 0.536378026008606, "rewards/rejected": -2.125223159790039, "step": 850 }, { "epoch": 0.1266009127042544, "grad_norm": 112.05360199697016, "learning_rate": 4.98931102019645e-07, "logits/chosen": -2.314889907836914, "logits/rejected": -2.2704579830169678, "logps/chosen": -329.2564392089844, "logps/rejected": -392.57867431640625, "loss": 0.5425, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.022720217704773, "rewards/margins": 0.9595369100570679, "rewards/rejected": -1.9822572469711304, "step": 860 }, { "epoch": 0.12807301634035037, "grad_norm": 90.50989484982753, "learning_rate": 4.988091335064037e-07, "logits/chosen": -2.290186643600464, "logits/rejected": -2.206688642501831, "logps/chosen": -394.3182067871094, "logps/rejected": -440.935546875, "loss": 0.6308, "rewards/accuracies": 0.7333332300186157, "rewards/chosen": -1.573823094367981, "rewards/margins": 0.5901575088500977, "rewards/rejected": -2.163980722427368, "step": 870 }, { "epoch": 0.12954511997644635, "grad_norm": 58.388436790562935, "learning_rate": 4.98680593617171e-07, "logits/chosen": -2.488431453704834, "logits/rejected": -2.4395880699157715, "logps/chosen": -386.14056396484375, "logps/rejected": -313.35498046875, "loss": 0.591, "rewards/accuracies": 0.5666666626930237, "rewards/chosen": -0.9596444964408875, "rewards/margins": 0.1092836856842041, "rewards/rejected": -1.0689282417297363, "step": 880 }, { "epoch": 0.1310172236125423, "grad_norm": 40.99851316434281, "learning_rate": 4.985454857468542e-07, "logits/chosen": -2.243323802947998, "logits/rejected": -2.22050404548645, "logps/chosen": -432.0625, "logps/rejected": -431.6446228027344, "loss": 0.6383, "rewards/accuracies": 0.5666667222976685, "rewards/chosen": -1.2183120250701904, "rewards/margins": 0.3802083432674408, "rewards/rejected": -1.5985205173492432, "step": 890 }, { "epoch": 0.1324893272486383, "grad_norm": 99.22369803083447, "learning_rate": 4.984038134638297e-07, "logits/chosen": -2.307290554046631, "logits/rejected": -2.3129024505615234, "logps/chosen": -355.7754821777344, "logps/rejected": -406.7025146484375, "loss": 0.5693, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.0389937162399292, "rewards/margins": 0.5364495515823364, "rewards/rejected": -1.5754432678222656, "step": 900 }, { "epoch": 0.13396143088473428, "grad_norm": 68.23159256155962, "learning_rate": 4.982555805098483e-07, "logits/chosen": -2.2068724632263184, "logits/rejected": -2.078933000564575, "logps/chosen": -337.3310546875, "logps/rejected": -306.7698669433594, "loss": 0.5051, "rewards/accuracies": 0.5999999642372131, "rewards/chosen": -0.9214721918106079, "rewards/margins": 0.4675617814064026, "rewards/rejected": -1.3890339136123657, "step": 910 }, { "epoch": 0.13543353452083026, "grad_norm": 182.81778083629945, "learning_rate": 4.981007907999372e-07, "logits/chosen": -2.19437837600708, "logits/rejected": -2.253805637359619, "logps/chosen": -298.8421936035156, "logps/rejected": -415.520751953125, "loss": 0.6246, "rewards/accuracies": 0.5333333611488342, "rewards/chosen": -1.0081114768981934, "rewards/margins": 0.34868913888931274, "rewards/rejected": -1.3568007946014404, "step": 920 }, { "epoch": 0.13690563815692625, "grad_norm": 87.66168135410162, "learning_rate": 4.979394484222961e-07, "logits/chosen": -2.0963408946990967, "logits/rejected": -2.127100944519043, "logps/chosen": -277.10980224609375, "logps/rejected": -451.10845947265625, "loss": 0.5668, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -0.9092451333999634, "rewards/margins": 0.9224964380264282, "rewards/rejected": -1.8317416906356812, "step": 930 }, { "epoch": 0.13837774179302223, "grad_norm": 113.42064742344525, "learning_rate": 4.977715576381888e-07, "logits/chosen": -2.062704563140869, "logits/rejected": -2.0764923095703125, "logps/chosen": -390.50482177734375, "logps/rejected": -448.83880615234375, "loss": 0.5976, "rewards/accuracies": 0.7333332300186157, "rewards/chosen": -1.1084699630737305, "rewards/margins": 0.6334068775177002, "rewards/rejected": -1.7418768405914307, "step": 940 }, { "epoch": 0.1398498454291182, "grad_norm": 84.98755404610068, "learning_rate": 4.975971228818315e-07, "logits/chosen": -2.2141575813293457, "logits/rejected": -2.1957879066467285, "logps/chosen": -329.3928527832031, "logps/rejected": -461.7449645996094, "loss": 0.5695, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -1.0723673105239868, "rewards/margins": 1.2911356687545776, "rewards/rejected": -2.3635029792785645, "step": 950 }, { "epoch": 0.1413219490652142, "grad_norm": 95.3981684051296, "learning_rate": 4.974161487602753e-07, "logits/chosen": -2.1243934631347656, "logits/rejected": -2.1209025382995605, "logps/chosen": -384.853759765625, "logps/rejected": -452.0210876464844, "loss": 0.5195, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.4263676404953003, "rewards/margins": 0.6128391027450562, "rewards/rejected": -2.0392069816589355, "step": 960 }, { "epoch": 0.14279405270131018, "grad_norm": 113.96843076110557, "learning_rate": 4.972286400532842e-07, "logits/chosen": -2.0679757595062256, "logits/rejected": -2.025749683380127, "logps/chosen": -413.76727294921875, "logps/rejected": -534.8135375976562, "loss": 0.5322, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.8679730892181396, "rewards/margins": 0.9302923083305359, "rewards/rejected": -2.7982656955718994, "step": 970 }, { "epoch": 0.14426615633740617, "grad_norm": 73.85476043164623, "learning_rate": 4.970346017132097e-07, "logits/chosen": -2.091668128967285, "logits/rejected": -1.8952341079711914, "logps/chosen": -463.33251953125, "logps/rejected": -512.8320922851562, "loss": 0.5526, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.806361436843872, "rewards/margins": 1.1053744554519653, "rewards/rejected": -2.9117355346679688, "step": 980 }, { "epoch": 0.14573825997350212, "grad_norm": 76.9679999494502, "learning_rate": 4.96834038864859e-07, "logits/chosen": -2.069716215133667, "logits/rejected": -2.039652109146118, "logps/chosen": -477.3058166503906, "logps/rejected": -492.8099060058594, "loss": 0.4873, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -1.745233178138733, "rewards/margins": 0.6889790296554565, "rewards/rejected": -2.4342122077941895, "step": 990 }, { "epoch": 0.1472103636095981, "grad_norm": 72.1161888821459, "learning_rate": 4.966269568053605e-07, "logits/chosen": -2.1215076446533203, "logits/rejected": -2.054935932159424, "logps/chosen": -439.8961486816406, "logps/rejected": -526.8071899414062, "loss": 0.5634, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.3309630155563354, "rewards/margins": 0.9732717275619507, "rewards/rejected": -2.3042349815368652, "step": 1000 }, { "epoch": 0.1486824672456941, "grad_norm": 136.2850007298125, "learning_rate": 4.964133610040232e-07, "logits/chosen": -1.9484182596206665, "logits/rejected": -1.8160221576690674, "logps/chosen": -418.21173095703125, "logps/rejected": -513.11767578125, "loss": 0.5841, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.6914863586425781, "rewards/margins": 0.8490256071090698, "rewards/rejected": -2.5405116081237793, "step": 1010 }, { "epoch": 0.15015457088179007, "grad_norm": 57.74832561411465, "learning_rate": 4.961932571021928e-07, "logits/chosen": -2.165980100631714, "logits/rejected": -2.040670871734619, "logps/chosen": -415.43768310546875, "logps/rejected": -455.57171630859375, "loss": 0.4488, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.5381168127059937, "rewards/margins": 0.6904627084732056, "rewards/rejected": -2.228579521179199, "step": 1020 }, { "epoch": 0.15162667451788606, "grad_norm": 81.18313465845735, "learning_rate": 4.959666509131025e-07, "logits/chosen": -2.122326374053955, "logits/rejected": -1.9732322692871094, "logps/chosen": -398.5821228027344, "logps/rejected": -463.12890625, "loss": 0.5122, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.3551348447799683, "rewards/margins": 1.0280563831329346, "rewards/rejected": -2.3831915855407715, "step": 1030 }, { "epoch": 0.15309877815398204, "grad_norm": 115.82876072399974, "learning_rate": 4.957335484217193e-07, "logits/chosen": -2.1244285106658936, "logits/rejected": -2.145282745361328, "logps/chosen": -412.422607421875, "logps/rejected": -458.42578125, "loss": 0.6047, "rewards/accuracies": 0.5333333015441895, "rewards/chosen": -1.257176160812378, "rewards/margins": 0.412966787815094, "rewards/rejected": -1.6701428890228271, "step": 1040 }, { "epoch": 0.15457088179007802, "grad_norm": 85.64937174321328, "learning_rate": 4.954939557845862e-07, "logits/chosen": -2.214097499847412, "logits/rejected": -2.086029529571533, "logps/chosen": -468.0782165527344, "logps/rejected": -518.0413208007812, "loss": 0.6009, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.4204020500183105, "rewards/margins": 0.45726943016052246, "rewards/rejected": -1.8776715993881226, "step": 1050 }, { "epoch": 0.156042985426174, "grad_norm": 76.16188074968915, "learning_rate": 4.952478793296594e-07, "logits/chosen": -1.9984462261199951, "logits/rejected": -2.0089707374572754, "logps/chosen": -291.55047607421875, "logps/rejected": -413.54534912109375, "loss": 0.4861, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -1.1199281215667725, "rewards/margins": 0.9721947908401489, "rewards/rejected": -2.092123031616211, "step": 1060 }, { "epoch": 0.15751508906227, "grad_norm": 66.89251406016741, "learning_rate": 4.949953255561411e-07, "logits/chosen": -2.1272060871124268, "logits/rejected": -2.082313060760498, "logps/chosen": -413.8898010253906, "logps/rejected": -489.11163330078125, "loss": 0.4714, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.1104748249053955, "rewards/margins": 1.042687177658081, "rewards/rejected": -2.1531615257263184, "step": 1070 }, { "epoch": 0.15898719269836598, "grad_norm": 62.60189813831605, "learning_rate": 4.947363011343083e-07, "logits/chosen": -2.0016605854034424, "logits/rejected": -1.9395275115966797, "logps/chosen": -431.47515869140625, "logps/rejected": -456.5384216308594, "loss": 0.4674, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.4982682466506958, "rewards/margins": 0.6886683106422424, "rewards/rejected": -2.186936855316162, "step": 1080 }, { "epoch": 0.16045929633446196, "grad_norm": 85.8807908568554, "learning_rate": 4.944708129053362e-07, "logits/chosen": -1.873248815536499, "logits/rejected": -1.8893684148788452, "logps/chosen": -446.6443786621094, "logps/rejected": -435.57025146484375, "loss": 0.5677, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.6512603759765625, "rewards/margins": 0.6568886041641235, "rewards/rejected": -2.3081488609313965, "step": 1090 }, { "epoch": 0.16193139997055792, "grad_norm": 91.37167049438412, "learning_rate": 4.941988678811176e-07, "logits/chosen": -1.877694845199585, "logits/rejected": -1.7398853302001953, "logps/chosen": -440.3699645996094, "logps/rejected": -484.8650817871094, "loss": 0.5656, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.721146583557129, "rewards/margins": 0.9025732278823853, "rewards/rejected": -2.6237196922302246, "step": 1100 }, { "epoch": 0.1634035036066539, "grad_norm": 70.1640832487081, "learning_rate": 4.939204732440777e-07, "logits/chosen": -1.9863033294677734, "logits/rejected": -1.9010241031646729, "logps/chosen": -374.4677734375, "logps/rejected": -389.5589294433594, "loss": 0.469, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.4531137943267822, "rewards/margins": 0.8062105178833008, "rewards/rejected": -2.259324312210083, "step": 1110 }, { "epoch": 0.16487560724274988, "grad_norm": 83.51661410069474, "learning_rate": 4.936356363469845e-07, "logits/chosen": -1.878106713294983, "logits/rejected": -1.8452297449111938, "logps/chosen": -426.2998962402344, "logps/rejected": -505.9771423339844, "loss": 0.6071, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.5932050943374634, "rewards/margins": 0.5873239040374756, "rewards/rejected": -2.1805288791656494, "step": 1120 }, { "epoch": 0.16634771087884587, "grad_norm": 194.64662554460307, "learning_rate": 4.933443647127546e-07, "logits/chosen": -1.9042288064956665, "logits/rejected": -1.8562898635864258, "logps/chosen": -391.3512268066406, "logps/rejected": -425.87518310546875, "loss": 0.585, "rewards/accuracies": 0.7333332300186157, "rewards/chosen": -1.4531123638153076, "rewards/margins": 0.6793515086174011, "rewards/rejected": -2.1324636936187744, "step": 1130 }, { "epoch": 0.16781981451494185, "grad_norm": 150.4762925491126, "learning_rate": 4.930466660342543e-07, "logits/chosen": -1.6633784770965576, "logits/rejected": -1.5810251235961914, "logps/chosen": -470.38555908203125, "logps/rejected": -486.028076171875, "loss": 0.6363, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -2.476055860519409, "rewards/margins": 0.576412558555603, "rewards/rejected": -3.0524685382843018, "step": 1140 }, { "epoch": 0.16929191815103783, "grad_norm": 1059.8004323037026, "learning_rate": 4.927425481740968e-07, "logits/chosen": -1.704079031944275, "logits/rejected": -1.6063989400863647, "logps/chosen": -494.673095703125, "logps/rejected": -524.99755859375, "loss": 0.6717, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -2.659291982650757, "rewards/margins": 0.5076649785041809, "rewards/rejected": -3.166957139968872, "step": 1150 }, { "epoch": 0.17076402178713382, "grad_norm": 92.60206409904589, "learning_rate": 4.924320191644341e-07, "logits/chosen": -1.8932876586914062, "logits/rejected": -1.8256971836090088, "logps/chosen": -526.1300048828125, "logps/rejected": -605.43017578125, "loss": 0.5503, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.274585008621216, "rewards/margins": 1.0169695615768433, "rewards/rejected": -3.2915549278259277, "step": 1160 }, { "epoch": 0.1722361254232298, "grad_norm": 117.8554758287773, "learning_rate": 4.921150872067452e-07, "logits/chosen": -1.9694545269012451, "logits/rejected": -1.8143869638442993, "logps/chosen": -479.6485290527344, "logps/rejected": -476.7415466308594, "loss": 0.5049, "rewards/accuracies": 0.5333333611488342, "rewards/chosen": -1.804478406906128, "rewards/margins": 0.553841233253479, "rewards/rejected": -2.3583197593688965, "step": 1170 }, { "epoch": 0.17370822905932579, "grad_norm": 287.3277348516987, "learning_rate": 4.917917606716194e-07, "logits/chosen": -1.7659416198730469, "logits/rejected": -1.7585029602050781, "logps/chosen": -375.60015869140625, "logps/rejected": -470.989013671875, "loss": 0.5279, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.9602396488189697, "rewards/margins": 0.42130643129348755, "rewards/rejected": -2.3815460205078125, "step": 1180 }, { "epoch": 0.17518033269542177, "grad_norm": 52.304859212281535, "learning_rate": 4.914620480985352e-07, "logits/chosen": -1.9349756240844727, "logits/rejected": -1.9279472827911377, "logps/chosen": -495.8687438964844, "logps/rejected": -537.65283203125, "loss": 0.5704, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.5462409257888794, "rewards/margins": 0.8103898763656616, "rewards/rejected": -2.356630563735962, "step": 1190 }, { "epoch": 0.17665243633151773, "grad_norm": 91.8676158422065, "learning_rate": 4.911259581956345e-07, "logits/chosen": -1.833195686340332, "logits/rejected": -1.9040639400482178, "logps/chosen": -432.99395751953125, "logps/rejected": -610.0288696289062, "loss": 0.6933, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.5811357498168945, "rewards/margins": 0.8339434862136841, "rewards/rejected": -2.415079116821289, "step": 1200 }, { "epoch": 0.1781245399676137, "grad_norm": 79.47387367101264, "learning_rate": 4.907834998394932e-07, "logits/chosen": -1.9623525142669678, "logits/rejected": -1.9679954051971436, "logps/chosen": -290.91845703125, "logps/rejected": -416.90032958984375, "loss": 0.4977, "rewards/accuracies": 0.9333332777023315, "rewards/chosen": -0.8924083709716797, "rewards/margins": 0.9155305027961731, "rewards/rejected": -1.8079389333724976, "step": 1210 }, { "epoch": 0.1795966436037097, "grad_norm": 96.05780834655081, "learning_rate": 4.904346820748862e-07, "logits/chosen": -1.9771655797958374, "logits/rejected": -1.8380553722381592, "logps/chosen": -442.7921447753906, "logps/rejected": -441.2320861816406, "loss": 0.5562, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.5602648258209229, "rewards/margins": 0.7709900140762329, "rewards/rejected": -2.331254720687866, "step": 1220 }, { "epoch": 0.18106874723980568, "grad_norm": 192.01556582915327, "learning_rate": 4.900795141145487e-07, "logits/chosen": -1.9421284198760986, "logits/rejected": -1.789272665977478, "logps/chosen": -334.18157958984375, "logps/rejected": -461.389404296875, "loss": 0.4702, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -1.4846737384796143, "rewards/margins": 0.8886607885360718, "rewards/rejected": -2.3733344078063965, "step": 1230 }, { "epoch": 0.18254085087590166, "grad_norm": 255.85495023538195, "learning_rate": 4.897180053389332e-07, "logits/chosen": -1.9627825021743774, "logits/rejected": -1.8504955768585205, "logps/chosen": -432.42242431640625, "logps/rejected": -501.09393310546875, "loss": 0.5592, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.9151782989501953, "rewards/margins": 0.9223679304122925, "rewards/rejected": -2.8375463485717773, "step": 1240 }, { "epoch": 0.18401295451199765, "grad_norm": 100.32293673239501, "learning_rate": 4.89350165295961e-07, "logits/chosen": -1.9722315073013306, "logits/rejected": -2.004082441329956, "logps/chosen": -436.787109375, "logps/rejected": -542.9174194335938, "loss": 0.5656, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.8804203271865845, "rewards/margins": 0.7965079545974731, "rewards/rejected": -2.6769282817840576, "step": 1250 }, { "epoch": 0.18548505814809363, "grad_norm": 108.56990565880037, "learning_rate": 4.88976003700771e-07, "logits/chosen": -2.0546443462371826, "logits/rejected": -1.9407790899276733, "logps/chosen": -571.1334228515625, "logps/rejected": -592.9005126953125, "loss": 0.5734, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.21150541305542, "rewards/margins": 0.8162603378295898, "rewards/rejected": -3.027766227722168, "step": 1260 }, { "epoch": 0.1869571617841896, "grad_norm": 576.7575925067408, "learning_rate": 4.885955304354622e-07, "logits/chosen": -1.8984283208847046, "logits/rejected": -1.8702316284179688, "logps/chosen": -377.49481201171875, "logps/rejected": -463.9615173339844, "loss": 0.5305, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.3347193002700806, "rewards/margins": 0.9362874031066895, "rewards/rejected": -2.2710065841674805, "step": 1270 }, { "epoch": 0.1884292654202856, "grad_norm": 73.36266299470105, "learning_rate": 4.882087555488331e-07, "logits/chosen": -2.1328797340393066, "logits/rejected": -2.031864881515503, "logps/chosen": -429.73828125, "logps/rejected": -418.3163146972656, "loss": 0.5371, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.1759955883026123, "rewards/margins": 0.7771323919296265, "rewards/rejected": -1.9531276226043701, "step": 1280 }, { "epoch": 0.18990136905638158, "grad_norm": 68.02291613118399, "learning_rate": 4.878156892561167e-07, "logits/chosen": -2.009929895401001, "logits/rejected": -1.9555679559707642, "logps/chosen": -360.5475158691406, "logps/rejected": -464.4381408691406, "loss": 0.5558, "rewards/accuracies": 0.5, "rewards/chosen": -1.6077797412872314, "rewards/margins": 0.6679785251617432, "rewards/rejected": -2.2757582664489746, "step": 1290 }, { "epoch": 0.19137347269247756, "grad_norm": 168.9729209419952, "learning_rate": 4.874163419387099e-07, "logits/chosen": -1.9611870050430298, "logits/rejected": -1.9176820516586304, "logps/chosen": -349.78619384765625, "logps/rejected": -480.7174377441406, "loss": 0.5364, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.621067762374878, "rewards/margins": 0.984261155128479, "rewards/rejected": -2.6053287982940674, "step": 1300 }, { "epoch": 0.19284557632857352, "grad_norm": 112.89657841963704, "learning_rate": 4.870107241438999e-07, "logits/chosen": -1.9834076166152954, "logits/rejected": -1.9171555042266846, "logps/chosen": -473.7816467285156, "logps/rejected": -575.8986206054688, "loss": 0.6297, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -2.2061500549316406, "rewards/margins": 1.0230388641357422, "rewards/rejected": -3.229188919067383, "step": 1310 }, { "epoch": 0.1943176799646695, "grad_norm": 106.24775303317578, "learning_rate": 4.865988465845852e-07, "logits/chosen": -1.9366060495376587, "logits/rejected": -1.8908555507659912, "logps/chosen": -482.27618408203125, "logps/rejected": -524.3118286132812, "loss": 0.5819, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -2.2628026008605957, "rewards/margins": 0.5921173095703125, "rewards/rejected": -2.854919910430908, "step": 1320 }, { "epoch": 0.1957897836007655, "grad_norm": 52.229401388486096, "learning_rate": 4.861807201389933e-07, "logits/chosen": -2.0587615966796875, "logits/rejected": -1.9710086584091187, "logps/chosen": -440.68341064453125, "logps/rejected": -486.0523986816406, "loss": 0.5216, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.7683641910552979, "rewards/margins": 0.9764081835746765, "rewards/rejected": -2.7447726726531982, "step": 1330 }, { "epoch": 0.19726188723686147, "grad_norm": 102.49039584777856, "learning_rate": 4.857563558503925e-07, "logits/chosen": -1.9479601383209229, "logits/rejected": -1.9149713516235352, "logps/chosen": -427.7066955566406, "logps/rejected": -524.1619873046875, "loss": 0.5739, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.6376683712005615, "rewards/margins": 0.7223914861679077, "rewards/rejected": -2.3600597381591797, "step": 1340 }, { "epoch": 0.19873399087295746, "grad_norm": 86.03021132265596, "learning_rate": 4.853257649268014e-07, "logits/chosen": -1.8271957635879517, "logits/rejected": -1.7711585760116577, "logps/chosen": -387.6052551269531, "logps/rejected": -394.2628173828125, "loss": 0.5485, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.6212457418441772, "rewards/margins": 0.5990225672721863, "rewards/rejected": -2.2202682495117188, "step": 1350 }, { "epoch": 0.20020609450905344, "grad_norm": 95.9549443531512, "learning_rate": 4.848889587406915e-07, "logits/chosen": -1.9340747594833374, "logits/rejected": -2.000556468963623, "logps/chosen": -424.41619873046875, "logps/rejected": -503.3523864746094, "loss": 0.5768, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.6836246252059937, "rewards/margins": 0.7771691083908081, "rewards/rejected": -2.460793972015381, "step": 1360 }, { "epoch": 0.20167819814514942, "grad_norm": 66.58619020064566, "learning_rate": 4.84445948828688e-07, "logits/chosen": -1.7994775772094727, "logits/rejected": -1.6953080892562866, "logps/chosen": -459.05084228515625, "logps/rejected": -351.1479187011719, "loss": 0.7021, "rewards/accuracies": 0.46666663885116577, "rewards/chosen": -1.83424973487854, "rewards/margins": 0.1528480052947998, "rewards/rejected": -1.9870975017547607, "step": 1370 }, { "epoch": 0.2031503017812454, "grad_norm": 39.301434660219456, "learning_rate": 4.839967468912645e-07, "logits/chosen": -2.0798094272613525, "logits/rejected": -2.029099464416504, "logps/chosen": -426.1573181152344, "logps/rejected": -466.2427673339844, "loss": 0.5844, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.2144460678100586, "rewards/margins": 0.6685677766799927, "rewards/rejected": -1.8830139636993408, "step": 1380 }, { "epoch": 0.2046224054173414, "grad_norm": 78.71981934177275, "learning_rate": 4.83541364792434e-07, "logits/chosen": -1.8970882892608643, "logits/rejected": -1.9339784383773804, "logps/chosen": -379.80426025390625, "logps/rejected": -446.1246032714844, "loss": 0.5298, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.593406319618225, "rewards/margins": 0.5336878895759583, "rewards/rejected": -2.127094268798828, "step": 1390 }, { "epoch": 0.20609450905343737, "grad_norm": 107.64488455114477, "learning_rate": 4.83079814559436e-07, "logits/chosen": -1.9898523092269897, "logits/rejected": -1.8959048986434937, "logps/chosen": -463.18060302734375, "logps/rejected": -483.04534912109375, "loss": 0.5633, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.7604682445526123, "rewards/margins": 0.6334084868431091, "rewards/rejected": -2.393876552581787, "step": 1400 }, { "epoch": 0.20756661268953333, "grad_norm": 57.38521937426867, "learning_rate": 4.826121083824181e-07, "logits/chosen": -1.8779270648956299, "logits/rejected": -1.8601785898208618, "logps/chosen": -461.5616760253906, "logps/rejected": -491.1333923339844, "loss": 0.5071, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.7569482326507568, "rewards/margins": 0.6989758610725403, "rewards/rejected": -2.4559242725372314, "step": 1410 }, { "epoch": 0.20903871632562931, "grad_norm": 70.40701427850496, "learning_rate": 4.82138258614115e-07, "logits/chosen": -1.9726396799087524, "logits/rejected": -1.910292625427246, "logps/chosen": -437.85479736328125, "logps/rejected": -490.97698974609375, "loss": 0.5426, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.5512851476669312, "rewards/margins": 0.6094131469726562, "rewards/rejected": -2.160698175430298, "step": 1420 }, { "epoch": 0.2105108199617253, "grad_norm": 68.54955508063232, "learning_rate": 4.816582777695212e-07, "logits/chosen": -2.0624613761901855, "logits/rejected": -1.977168321609497, "logps/chosen": -336.0431213378906, "logps/rejected": -433.075439453125, "loss": 0.5154, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.1242858171463013, "rewards/margins": 0.7905791401863098, "rewards/rejected": -1.9148648977279663, "step": 1430 }, { "epoch": 0.21198292359782128, "grad_norm": 53.662406903457814, "learning_rate": 4.811721785255612e-07, "logits/chosen": -1.8410117626190186, "logits/rejected": -1.9229590892791748, "logps/chosen": -330.0913391113281, "logps/rejected": -383.91876220703125, "loss": 0.5438, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.4411814212799072, "rewards/margins": 0.48734864592552185, "rewards/rejected": -1.9285300970077515, "step": 1440 }, { "epoch": 0.21345502723391727, "grad_norm": 99.78983887665171, "learning_rate": 4.806799737207546e-07, "logits/chosen": -1.7521324157714844, "logits/rejected": -1.763819932937622, "logps/chosen": -397.3249206542969, "logps/rejected": -485.8280334472656, "loss": 0.5558, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.8569271564483643, "rewards/margins": 0.6991360187530518, "rewards/rejected": -2.556062936782837, "step": 1450 }, { "epoch": 0.21492713087001325, "grad_norm": 77.4751921400414, "learning_rate": 4.801816763548766e-07, "logits/chosen": -2.006967067718506, "logits/rejected": -1.9145374298095703, "logps/chosen": -424.72161865234375, "logps/rejected": -503.0011291503906, "loss": 0.5403, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.3916488885879517, "rewards/margins": 0.8669332265853882, "rewards/rejected": -2.258582353591919, "step": 1460 }, { "epoch": 0.21639923450610923, "grad_norm": 106.09643165111075, "learning_rate": 4.796772995886151e-07, "logits/chosen": -1.9247760772705078, "logits/rejected": -1.9768764972686768, "logps/chosen": -395.3624572753906, "logps/rejected": -427.5353088378906, "loss": 0.5898, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.313778042793274, "rewards/margins": 0.5099066495895386, "rewards/rejected": -1.8236846923828125, "step": 1470 }, { "epoch": 0.21787133814220522, "grad_norm": 164.85826302580128, "learning_rate": 4.791668567432229e-07, "logits/chosen": -1.9292771816253662, "logits/rejected": -1.7980287075042725, "logps/chosen": -425.4931640625, "logps/rejected": -439.97747802734375, "loss": 0.616, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.306755781173706, "rewards/margins": 0.6289185285568237, "rewards/rejected": -1.9356744289398193, "step": 1480 }, { "epoch": 0.2193434417783012, "grad_norm": 145.10508377533765, "learning_rate": 4.78650361300166e-07, "logits/chosen": -1.8893663883209229, "logits/rejected": -1.882112741470337, "logps/chosen": -441.1393127441406, "logps/rejected": -517.1280517578125, "loss": 0.4683, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -1.0786323547363281, "rewards/margins": 0.9214528799057007, "rewards/rejected": -2.0000853538513184, "step": 1490 }, { "epoch": 0.22081554541439719, "grad_norm": 198.00339167625546, "learning_rate": 4.781278269007675e-07, "logits/chosen": -1.9149973392486572, "logits/rejected": -1.761517882347107, "logps/chosen": -499.5852966308594, "logps/rejected": -563.583984375, "loss": 0.4889, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.650298833847046, "rewards/margins": 1.0565168857574463, "rewards/rejected": -2.7068159580230713, "step": 1500 }, { "epoch": 0.22228764905049314, "grad_norm": 49.036813921932804, "learning_rate": 4.775992673458469e-07, "logits/chosen": -1.8019440174102783, "logits/rejected": -1.7608954906463623, "logps/chosen": -461.46319580078125, "logps/rejected": -509.35504150390625, "loss": 0.4643, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -1.8942543268203735, "rewards/margins": 0.8434314727783203, "rewards/rejected": -2.7376856803894043, "step": 1510 }, { "epoch": 0.22375975268658913, "grad_norm": 47.1731224324246, "learning_rate": 4.770646965953564e-07, "logits/chosen": -1.7631785869598389, "logits/rejected": -1.7487800121307373, "logps/chosen": -428.530517578125, "logps/rejected": -518.99365234375, "loss": 0.6563, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.326993703842163, "rewards/margins": 1.1345717906951904, "rewards/rejected": -2.4615654945373535, "step": 1520 }, { "epoch": 0.2252318563226851, "grad_norm": 90.82425083212647, "learning_rate": 4.765241287680116e-07, "logits/chosen": -2.0718226432800293, "logits/rejected": -1.8782758712768555, "logps/chosen": -428.19366455078125, "logps/rejected": -390.31036376953125, "loss": 0.512, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -0.919861912727356, "rewards/margins": 0.9122627973556519, "rewards/rejected": -1.8321247100830078, "step": 1530 }, { "epoch": 0.2267039599587811, "grad_norm": 99.4352748279063, "learning_rate": 4.759775781409187e-07, "logits/chosen": -1.95816171169281, "logits/rejected": -1.8278312683105469, "logps/chosen": -447.49530029296875, "logps/rejected": -474.17919921875, "loss": 0.562, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -1.7144454717636108, "rewards/margins": 0.7575380206108093, "rewards/rejected": -2.4719834327697754, "step": 1540 }, { "epoch": 0.22817606359487708, "grad_norm": 70.96061670177698, "learning_rate": 4.7542505914919775e-07, "logits/chosen": -1.9224542379379272, "logits/rejected": -1.9225962162017822, "logps/chosen": -454.31597900390625, "logps/rejected": -521.6945190429688, "loss": 0.4991, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.6472991704940796, "rewards/margins": 1.0311214923858643, "rewards/rejected": -2.6784205436706543, "step": 1550 }, { "epoch": 0.22964816723097306, "grad_norm": 225.93570983559735, "learning_rate": 4.7486658638560076e-07, "logits/chosen": -1.9805772304534912, "logits/rejected": -2.0407538414001465, "logps/chosen": -398.05804443359375, "logps/rejected": -469.94061279296875, "loss": 0.5792, "rewards/accuracies": 0.7333332300186157, "rewards/chosen": -1.4981138706207275, "rewards/margins": 0.6817944645881653, "rewards/rejected": -2.179908275604248, "step": 1560 }, { "epoch": 0.23112027086706904, "grad_norm": 75.77626864521106, "learning_rate": 4.7430217460012703e-07, "logits/chosen": -1.89935302734375, "logits/rejected": -1.8430445194244385, "logps/chosen": -416.69219970703125, "logps/rejected": -476.912353515625, "loss": 0.5618, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.4287060499191284, "rewards/margins": 1.1412321329116821, "rewards/rejected": -2.5699384212493896, "step": 1570 }, { "epoch": 0.23259237450316503, "grad_norm": 75.35179639832687, "learning_rate": 4.7373183869963295e-07, "logits/chosen": -2.1149773597717285, "logits/rejected": -2.0849950313568115, "logps/chosen": -438.9716796875, "logps/rejected": -464.542724609375, "loss": 0.5682, "rewards/accuracies": 0.7000000476837158, "rewards/chosen": -1.0148811340332031, "rewards/margins": 0.6320945024490356, "rewards/rejected": -1.6469755172729492, "step": 1580 }, { "epoch": 0.234064478139261, "grad_norm": 129.63107922689574, "learning_rate": 4.7315559374743896e-07, "logits/chosen": -2.100356101989746, "logits/rejected": -2.1032233238220215, "logps/chosen": -368.68450927734375, "logps/rejected": -456.0458068847656, "loss": 0.5552, "rewards/accuracies": 0.6333332657814026, "rewards/chosen": -1.142398476600647, "rewards/margins": 0.6042104959487915, "rewards/rejected": -1.746609091758728, "step": 1590 }, { "epoch": 0.235536581775357, "grad_norm": 61.71219474707247, "learning_rate": 4.725734549629308e-07, "logits/chosen": -1.8276703357696533, "logits/rejected": -1.7567144632339478, "logps/chosen": -372.30694580078125, "logps/rejected": -504.8480529785156, "loss": 0.5147, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.3657010793685913, "rewards/margins": 1.2297759056091309, "rewards/rejected": -2.5954768657684326, "step": 1600 }, { "epoch": 0.23700868541145298, "grad_norm": 81.0493165852162, "learning_rate": 4.719854377211585e-07, "logits/chosen": -1.9518165588378906, "logits/rejected": -1.951531171798706, "logps/chosen": -387.6022033691406, "logps/rejected": -385.2184143066406, "loss": 0.5937, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.3865382671356201, "rewards/margins": 0.5220063328742981, "rewards/rejected": -1.9085447788238525, "step": 1610 }, { "epoch": 0.23848078904754894, "grad_norm": 60.09382845884993, "learning_rate": 4.713915575524296e-07, "logits/chosen": -2.0600926876068115, "logits/rejected": -2.0641098022460938, "logps/chosen": -313.11114501953125, "logps/rejected": -348.462158203125, "loss": 0.6013, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.1776988506317139, "rewards/margins": 0.34402555227279663, "rewards/rejected": -1.5217244625091553, "step": 1620 }, { "epoch": 0.23995289268364492, "grad_norm": 55.96151058915186, "learning_rate": 4.7079183014189937e-07, "logits/chosen": -1.9451329708099365, "logits/rejected": -1.8646180629730225, "logps/chosen": -323.1206970214844, "logps/rejected": -387.49273681640625, "loss": 0.5703, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -0.9625275731086731, "rewards/margins": 0.8532989621162415, "rewards/rejected": -1.815826654434204, "step": 1630 }, { "epoch": 0.2414249963197409, "grad_norm": 75.74723272789755, "learning_rate": 4.7018627132915634e-07, "logits/chosen": -1.9486162662506104, "logits/rejected": -1.874757170677185, "logps/chosen": -378.52557373046875, "logps/rejected": -382.94622802734375, "loss": 0.528, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.0989223718643188, "rewards/margins": 0.5968096256256104, "rewards/rejected": -1.6957321166992188, "step": 1640 }, { "epoch": 0.2428970999558369, "grad_norm": 89.07340706477274, "learning_rate": 4.695748971078042e-07, "logits/chosen": -1.9029546976089478, "logits/rejected": -1.7417056560516357, "logps/chosen": -391.79150390625, "logps/rejected": -363.08538818359375, "loss": 0.5185, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.0512101650238037, "rewards/margins": 0.5672429800033569, "rewards/rejected": -1.6184532642364502, "step": 1650 }, { "epoch": 0.24436920359193287, "grad_norm": 50.4856901474438, "learning_rate": 4.689577236250389e-07, "logits/chosen": -1.6937227249145508, "logits/rejected": -1.5301963090896606, "logps/chosen": -408.7888488769531, "logps/rejected": -470.7008361816406, "loss": 0.4977, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.4841692447662354, "rewards/margins": 0.727135956287384, "rewards/rejected": -2.2113051414489746, "step": 1660 }, { "epoch": 0.24584130722802885, "grad_norm": 82.14693999635138, "learning_rate": 4.683347671812228e-07, "logits/chosen": -1.7227070331573486, "logits/rejected": -1.527448296546936, "logps/chosen": -418.6912536621094, "logps/rejected": -401.7694396972656, "loss": 0.5397, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -1.6010897159576416, "rewards/margins": 0.7377594709396362, "rewards/rejected": -2.3388493061065674, "step": 1670 }, { "epoch": 0.24731341086412484, "grad_norm": 183.94121321876028, "learning_rate": 4.677060442294537e-07, "logits/chosen": -1.6719534397125244, "logits/rejected": -1.6455955505371094, "logps/chosen": -524.6824951171875, "logps/rejected": -547.9179077148438, "loss": 0.5941, "rewards/accuracies": 0.5666667222976685, "rewards/chosen": -2.1113953590393066, "rewards/margins": 0.4821031987667084, "rewards/rejected": -2.593498468399048, "step": 1680 }, { "epoch": 0.24878551450022082, "grad_norm": 75.5082371575327, "learning_rate": 4.6707157137513056e-07, "logits/chosen": -1.74550461769104, "logits/rejected": -1.7400407791137695, "logps/chosen": -470.6490173339844, "logps/rejected": -461.89605712890625, "loss": 0.6451, "rewards/accuracies": 0.6333333849906921, "rewards/chosen": -1.9410581588745117, "rewards/margins": 0.4430798888206482, "rewards/rejected": -2.3841381072998047, "step": 1690 }, { "epoch": 0.2502576181363168, "grad_norm": 98.82772188339317, "learning_rate": 4.664313653755147e-07, "logits/chosen": -1.9526565074920654, "logits/rejected": -1.883766770362854, "logps/chosen": -564.9190063476562, "logps/rejected": -515.51806640625, "loss": 0.5262, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.6935169696807861, "rewards/margins": 0.49616527557373047, "rewards/rejected": -2.1896822452545166, "step": 1700 }, { "epoch": 0.25172972177241276, "grad_norm": 101.66239188256434, "learning_rate": 4.6578544313928735e-07, "logits/chosen": -1.4847203493118286, "logits/rejected": -1.3912432193756104, "logps/chosen": -449.73388671875, "logps/rejected": -569.7845458984375, "loss": 0.5128, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.9837766885757446, "rewards/margins": 1.1695940494537354, "rewards/rejected": -3.1533703804016113, "step": 1710 }, { "epoch": 0.2532018254085088, "grad_norm": 71.15192406825254, "learning_rate": 4.6513382172610324e-07, "logits/chosen": -1.4488019943237305, "logits/rejected": -1.2987825870513916, "logps/chosen": -476.8401794433594, "logps/rejected": -549.9502563476562, "loss": 0.5701, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.4032297134399414, "rewards/margins": 0.8704252243041992, "rewards/rejected": -3.2736549377441406, "step": 1720 }, { "epoch": 0.25467392904460473, "grad_norm": 115.6657540517164, "learning_rate": 4.6447651834613955e-07, "logits/chosen": -1.5709898471832275, "logits/rejected": -1.5908291339874268, "logps/chosen": -520.153564453125, "logps/rejected": -584.2728271484375, "loss": 0.6331, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.6676547527313232, "rewards/margins": 0.5557428598403931, "rewards/rejected": -3.223397731781006, "step": 1730 }, { "epoch": 0.25614603268070074, "grad_norm": 75.3319050439407, "learning_rate": 4.638135503596419e-07, "logits/chosen": -1.6912071704864502, "logits/rejected": -1.7145217657089233, "logps/chosen": -448.5433654785156, "logps/rejected": -559.8304443359375, "loss": 0.5967, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.9018909931182861, "rewards/margins": 0.6624494194984436, "rewards/rejected": -2.564340353012085, "step": 1740 }, { "epoch": 0.2576181363167967, "grad_norm": 135.1266647777451, "learning_rate": 4.6314493527646553e-07, "logits/chosen": -1.6531956195831299, "logits/rejected": -1.5040241479873657, "logps/chosen": -614.0111083984375, "logps/rejected": -600.8482666015625, "loss": 0.5588, "rewards/accuracies": 0.5333333015441895, "rewards/chosen": -2.88460636138916, "rewards/margins": 0.49871939420700073, "rewards/rejected": -3.3833255767822266, "step": 1750 }, { "epoch": 0.2590902399528927, "grad_norm": 229.84979404838685, "learning_rate": 4.624706907556129e-07, "logits/chosen": -1.535413384437561, "logits/rejected": -1.443834662437439, "logps/chosen": -490.82684326171875, "logps/rejected": -513.51220703125, "loss": 0.6302, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -2.540764570236206, "rewards/margins": 0.6940882802009583, "rewards/rejected": -3.2348525524139404, "step": 1760 }, { "epoch": 0.26056234358898867, "grad_norm": 102.39544835337084, "learning_rate": 4.617908346047673e-07, "logits/chosen": -1.5806939601898193, "logits/rejected": -1.4859967231750488, "logps/chosen": -411.6058044433594, "logps/rejected": -524.4972534179688, "loss": 0.6848, "rewards/accuracies": 0.6666667461395264, "rewards/chosen": -1.9826698303222656, "rewards/margins": 1.014257788658142, "rewards/rejected": -2.9969279766082764, "step": 1770 }, { "epoch": 0.2620344472250846, "grad_norm": 51.64233638006087, "learning_rate": 4.6110538477982265e-07, "logits/chosen": -1.85050368309021, "logits/rejected": -1.7250585556030273, "logps/chosen": -349.91790771484375, "logps/rejected": -487.35943603515625, "loss": 0.5408, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.2039263248443604, "rewards/margins": 1.247125506401062, "rewards/rejected": -2.451051712036133, "step": 1780 }, { "epoch": 0.26350655086118063, "grad_norm": 161.90036469442958, "learning_rate": 4.6041435938440887e-07, "logits/chosen": -1.9583768844604492, "logits/rejected": -1.820737600326538, "logps/chosen": -329.48724365234375, "logps/rejected": -395.01666259765625, "loss": 0.5561, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -0.8706947565078735, "rewards/margins": 0.616737961769104, "rewards/rejected": -1.4874327182769775, "step": 1790 }, { "epoch": 0.2649786544972766, "grad_norm": 68.37449519907922, "learning_rate": 4.5971777666941445e-07, "logits/chosen": -1.9366155862808228, "logits/rejected": -1.9282124042510986, "logps/chosen": -389.47503662109375, "logps/rejected": -463.842529296875, "loss": 0.629, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.09441077709198, "rewards/margins": 0.9559955596923828, "rewards/rejected": -2.0504064559936523, "step": 1800 }, { "epoch": 0.2664507581333726, "grad_norm": 58.94192935579279, "learning_rate": 4.5901565503250373e-07, "logits/chosen": -1.8732668161392212, "logits/rejected": -1.8085658550262451, "logps/chosen": -410.6376037597656, "logps/rejected": -477.9228515625, "loss": 0.4429, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -1.1377919912338257, "rewards/margins": 0.9135912656784058, "rewards/rejected": -2.0513832569122314, "step": 1810 }, { "epoch": 0.26792286176946856, "grad_norm": 106.17374938574147, "learning_rate": 4.583080130176312e-07, "logits/chosen": -1.6969807147979736, "logits/rejected": -1.5779601335525513, "logps/chosen": -423.90814208984375, "logps/rejected": -438.09228515625, "loss": 0.5464, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.6757621765136719, "rewards/margins": 0.6260987520217896, "rewards/rejected": -2.301860809326172, "step": 1820 }, { "epoch": 0.26939496540556457, "grad_norm": 58.151446689539824, "learning_rate": 4.575948693145518e-07, "logits/chosen": -1.5030003786087036, "logits/rejected": -1.5252426862716675, "logps/chosen": -478.30322265625, "logps/rejected": -531.8361206054688, "loss": 0.4875, "rewards/accuracies": 0.7000000476837158, "rewards/chosen": -1.7562452554702759, "rewards/margins": 0.7899385690689087, "rewards/rejected": -2.5461840629577637, "step": 1830 }, { "epoch": 0.2708670690416605, "grad_norm": 140.9049693185743, "learning_rate": 4.568762427583275e-07, "logits/chosen": -1.4599277973175049, "logits/rejected": -1.5029892921447754, "logps/chosen": -467.74530029296875, "logps/rejected": -566.2022705078125, "loss": 0.5047, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -2.079827070236206, "rewards/margins": 0.8469683527946472, "rewards/rejected": -2.926795244216919, "step": 1840 }, { "epoch": 0.27233917267775654, "grad_norm": 98.50078757323077, "learning_rate": 4.561521523288293e-07, "logits/chosen": -1.3252590894699097, "logits/rejected": -1.3201786279678345, "logps/chosen": -518.99072265625, "logps/rejected": -627.2073364257812, "loss": 0.5429, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.148075580596924, "rewards/margins": 1.0770463943481445, "rewards/rejected": -3.2251219749450684, "step": 1850 }, { "epoch": 0.2738112763138525, "grad_norm": 77.53821488417238, "learning_rate": 4.554226171502365e-07, "logits/chosen": -1.7430492639541626, "logits/rejected": -1.5309737920761108, "logps/chosen": -462.879638671875, "logps/rejected": -513.399169921875, "loss": 0.4981, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.619969367980957, "rewards/margins": 0.6844587922096252, "rewards/rejected": -2.3044278621673584, "step": 1860 }, { "epoch": 0.2752833799499485, "grad_norm": 47.10442635576484, "learning_rate": 4.546876564905313e-07, "logits/chosen": -1.619408369064331, "logits/rejected": -1.840978980064392, "logps/chosen": -378.6471862792969, "logps/rejected": -593.4290771484375, "loss": 0.5442, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.5935884714126587, "rewards/margins": 0.9168362617492676, "rewards/rejected": -2.5104246139526367, "step": 1870 }, { "epoch": 0.27675548358604446, "grad_norm": 273.2367553228704, "learning_rate": 4.5394728976099015e-07, "logits/chosen": -1.5420583486557007, "logits/rejected": -1.3446629047393799, "logps/chosen": -455.2611389160156, "logps/rejected": -630.8297119140625, "loss": 0.4648, "rewards/accuracies": 0.9000000953674316, "rewards/chosen": -1.963801383972168, "rewards/margins": 2.1008026599884033, "rewards/rejected": -4.06460428237915, "step": 1880 }, { "epoch": 0.2782275872221404, "grad_norm": 192.7634248185056, "learning_rate": 4.532015365156705e-07, "logits/chosen": -1.4305152893066406, "logits/rejected": -1.4293017387390137, "logps/chosen": -510.6460876464844, "logps/rejected": -551.4142456054688, "loss": 0.6284, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -2.2284276485443115, "rewards/margins": 0.8585487604141235, "rewards/rejected": -3.0869762897491455, "step": 1890 }, { "epoch": 0.2796996908582364, "grad_norm": 57.69637029955735, "learning_rate": 4.524504164508951e-07, "logits/chosen": -1.6792455911636353, "logits/rejected": -1.6008403301239014, "logps/chosen": -463.2054138183594, "logps/rejected": -509.00537109375, "loss": 0.5019, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.6925989389419556, "rewards/margins": 0.686219334602356, "rewards/rejected": -2.3788185119628906, "step": 1900 }, { "epoch": 0.2811717944943324, "grad_norm": 68.40972228659973, "learning_rate": 4.5169394940473137e-07, "logits/chosen": -1.7620881795883179, "logits/rejected": -1.6334224939346313, "logps/chosen": -453.60333251953125, "logps/rejected": -571.8778686523438, "loss": 0.511, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.689611792564392, "rewards/margins": 1.1878001689910889, "rewards/rejected": -2.8774123191833496, "step": 1910 }, { "epoch": 0.2826438981304284, "grad_norm": 108.28897894151297, "learning_rate": 4.509321553564676e-07, "logits/chosen": -1.6775577068328857, "logits/rejected": -1.6379661560058594, "logps/chosen": -444.2704162597656, "logps/rejected": -509.41546630859375, "loss": 0.5872, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.6652408838272095, "rewards/margins": 0.7498928308486938, "rewards/rejected": -2.4151339530944824, "step": 1920 }, { "epoch": 0.28411600176652435, "grad_norm": 78.5464549732301, "learning_rate": 4.501650544260848e-07, "logits/chosen": -1.5278265476226807, "logits/rejected": -1.6430070400238037, "logps/chosen": -345.4292297363281, "logps/rejected": -400.3189697265625, "loss": 0.5005, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.1476633548736572, "rewards/margins": 0.5958186388015747, "rewards/rejected": -1.743481993675232, "step": 1930 }, { "epoch": 0.28558810540262036, "grad_norm": 91.29964298945447, "learning_rate": 4.4939266687372636e-07, "logits/chosen": -1.608768105506897, "logits/rejected": -1.5200564861297607, "logps/chosen": -511.80303955078125, "logps/rejected": -502.6783142089844, "loss": 0.5877, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -1.6708977222442627, "rewards/margins": 0.807176411151886, "rewards/rejected": -2.478074312210083, "step": 1940 }, { "epoch": 0.2870602090387163, "grad_norm": 146.2232993455667, "learning_rate": 4.4861501309916185e-07, "logits/chosen": -1.72658371925354, "logits/rejected": -1.5450631380081177, "logps/chosen": -422.600341796875, "logps/rejected": -451.49737548828125, "loss": 0.4793, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.633721113204956, "rewards/margins": 1.042769193649292, "rewards/rejected": -2.676490306854248, "step": 1950 }, { "epoch": 0.28853231267481233, "grad_norm": 73.58047268290248, "learning_rate": 4.478321136412487e-07, "logits/chosen": -1.400095820426941, "logits/rejected": -1.4668046236038208, "logps/chosen": -502.67254638671875, "logps/rejected": -658.44140625, "loss": 0.6173, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.381251811981201, "rewards/margins": 1.2020304203033447, "rewards/rejected": -3.583281993865967, "step": 1960 }, { "epoch": 0.2900044163109083, "grad_norm": 45.42567695202515, "learning_rate": 4.4704398917738996e-07, "logits/chosen": -1.6204618215560913, "logits/rejected": -1.6879221200942993, "logps/chosen": -524.9888305664062, "logps/rejected": -639.3353271484375, "loss": 0.5938, "rewards/accuracies": 0.5666666626930237, "rewards/chosen": -2.5351343154907227, "rewards/margins": 0.9098974466323853, "rewards/rejected": -3.4450316429138184, "step": 1970 }, { "epoch": 0.29147651994700424, "grad_norm": 71.48299353606512, "learning_rate": 4.4625066052298766e-07, "logits/chosen": -1.615454912185669, "logits/rejected": -1.6648781299591064, "logps/chosen": -456.6764221191406, "logps/rejected": -564.7952880859375, "loss": 0.6478, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -2.189084768295288, "rewards/margins": 0.40208807587623596, "rewards/rejected": -2.591172933578491, "step": 1980 }, { "epoch": 0.29294862358310025, "grad_norm": 45.538931937805685, "learning_rate": 4.454521486308936e-07, "logits/chosen": -1.8534252643585205, "logits/rejected": -1.7413625717163086, "logps/chosen": -533.1121215820312, "logps/rejected": -546.1458129882812, "loss": 0.5436, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -2.2162203788757324, "rewards/margins": 0.5562283396720886, "rewards/rejected": -2.772448778152466, "step": 1990 }, { "epoch": 0.2944207272191962, "grad_norm": 76.26497187556322, "learning_rate": 4.4464847459085553e-07, "logits/chosen": -1.7512187957763672, "logits/rejected": -1.7058212757110596, "logps/chosen": -466.07733154296875, "logps/rejected": -524.7542724609375, "loss": 0.4741, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.9114131927490234, "rewards/margins": 0.738069474697113, "rewards/rejected": -2.6494827270507812, "step": 2000 }, { "epoch": 0.2958928308552922, "grad_norm": 98.0863216532731, "learning_rate": 4.438396596289604e-07, "logits/chosen": -1.6177749633789062, "logits/rejected": -1.5478427410125732, "logps/chosen": -444.4658203125, "logps/rejected": -530.3812255859375, "loss": 0.6137, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.7594554424285889, "rewards/margins": 1.1266467571258545, "rewards/rejected": -2.8861021995544434, "step": 2010 }, { "epoch": 0.2973649344913882, "grad_norm": 44.185656245031204, "learning_rate": 4.430257251070736e-07, "logits/chosen": -1.6525189876556396, "logits/rejected": -1.64784836769104, "logps/chosen": -414.99462890625, "logps/rejected": -457.1453552246094, "loss": 0.5189, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.8041388988494873, "rewards/margins": 0.7363736629486084, "rewards/rejected": -2.5405125617980957, "step": 2020 }, { "epoch": 0.2988370381274842, "grad_norm": 105.60349488334649, "learning_rate": 4.422066925222748e-07, "logits/chosen": -1.412907600402832, "logits/rejected": -1.3582699298858643, "logps/chosen": -414.4461975097656, "logps/rejected": -481.41436767578125, "loss": 0.5359, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.7802703380584717, "rewards/margins": 0.8613284826278687, "rewards/rejected": -2.641598701477051, "step": 2030 }, { "epoch": 0.30030914176358015, "grad_norm": 72.58853766247252, "learning_rate": 4.4138258350629033e-07, "logits/chosen": -1.8623470067977905, "logits/rejected": -1.5975165367126465, "logps/chosen": -466.7835388183594, "logps/rejected": -492.2386779785156, "loss": 0.5532, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.3512439727783203, "rewards/margins": 0.8019021153450012, "rewards/rejected": -2.153146266937256, "step": 2040 }, { "epoch": 0.30178124539967616, "grad_norm": 90.60349334079292, "learning_rate": 4.405534198249216e-07, "logits/chosen": -1.810738205909729, "logits/rejected": -1.5894935131072998, "logps/chosen": -369.27734375, "logps/rejected": -417.78399658203125, "loss": 0.513, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.2387871742248535, "rewards/margins": 0.8931953310966492, "rewards/rejected": -2.1319825649261475, "step": 2050 }, { "epoch": 0.3032533490357721, "grad_norm": 145.52787685203137, "learning_rate": 4.3971922337747045e-07, "logits/chosen": -1.7671382427215576, "logits/rejected": -1.7577235698699951, "logps/chosen": -387.68212890625, "logps/rejected": -431.3033142089844, "loss": 0.4633, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.1763545274734497, "rewards/margins": 0.864321231842041, "rewards/rejected": -2.0406758785247803, "step": 2060 }, { "epoch": 0.3047254526718681, "grad_norm": 128.78691357641787, "learning_rate": 4.388800161961606e-07, "logits/chosen": -1.8475401401519775, "logits/rejected": -1.5575668811798096, "logps/chosen": -488.7298278808594, "logps/rejected": -542.2572631835938, "loss": 0.4952, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.9894729852676392, "rewards/margins": 1.0202587842941284, "rewards/rejected": -3.0097317695617676, "step": 2070 }, { "epoch": 0.3061975563079641, "grad_norm": 100.22977386904307, "learning_rate": 4.380358204455559e-07, "logits/chosen": -1.5467193126678467, "logits/rejected": -1.4157942533493042, "logps/chosen": -437.54541015625, "logps/rejected": -563.5391235351562, "loss": 0.5542, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -2.265477180480957, "rewards/margins": 0.9674869775772095, "rewards/rejected": -3.232964038848877, "step": 2080 }, { "epoch": 0.30766965994406004, "grad_norm": 136.05047237428258, "learning_rate": 4.3718665842197494e-07, "logits/chosen": -1.4792824983596802, "logits/rejected": -1.2502820491790771, "logps/chosen": -435.041015625, "logps/rejected": -483.5919494628906, "loss": 0.5162, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.1088335514068604, "rewards/margins": 0.9096527099609375, "rewards/rejected": -3.018486499786377, "step": 2090 }, { "epoch": 0.30914176358015605, "grad_norm": 90.16904640691057, "learning_rate": 4.363325525529019e-07, "logits/chosen": -1.6123536825180054, "logits/rejected": -1.5360729694366455, "logps/chosen": -494.77874755859375, "logps/rejected": -537.6286010742188, "loss": 0.5353, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.2117531299591064, "rewards/margins": 0.9112815856933594, "rewards/rejected": -3.123034954071045, "step": 2100 }, { "epoch": 0.310613867216252, "grad_norm": 53.0978697251245, "learning_rate": 4.354735253963947e-07, "logits/chosen": -1.6993602514266968, "logits/rejected": -1.6057443618774414, "logps/chosen": -404.397216796875, "logps/rejected": -487.01544189453125, "loss": 0.5217, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.6291040182113647, "rewards/margins": 1.011840581893921, "rewards/rejected": -2.640944480895996, "step": 2110 }, { "epoch": 0.312085970852348, "grad_norm": 102.9771705233389, "learning_rate": 4.3460959964048854e-07, "logits/chosen": -1.8225218057632446, "logits/rejected": -1.703957200050354, "logps/chosen": -503.252197265625, "logps/rejected": -531.5034790039062, "loss": 0.6341, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -2.0066771507263184, "rewards/margins": 0.5090205073356628, "rewards/rejected": -2.515697717666626, "step": 2120 }, { "epoch": 0.31355807448844397, "grad_norm": 101.6792683943578, "learning_rate": 4.337407981025974e-07, "logits/chosen": -1.9180629253387451, "logits/rejected": -1.729644536972046, "logps/chosen": -505.4967346191406, "logps/rejected": -643.906005859375, "loss": 0.5094, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.9095513820648193, "rewards/margins": 1.203673005104065, "rewards/rejected": -3.113224506378174, "step": 2130 }, { "epoch": 0.31503017812454, "grad_norm": 80.68076892749208, "learning_rate": 4.3286714372891086e-07, "logits/chosen": -1.854379415512085, "logits/rejected": -1.736157774925232, "logps/chosen": -399.6598815917969, "logps/rejected": -480.1727600097656, "loss": 0.5159, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.61972177028656, "rewards/margins": 0.7012790441513062, "rewards/rejected": -2.321000814437866, "step": 2140 }, { "epoch": 0.31650228176063594, "grad_norm": 176.42924773487638, "learning_rate": 4.319886595937885e-07, "logits/chosen": -1.83878493309021, "logits/rejected": -1.7423222064971924, "logps/chosen": -476.2215881347656, "logps/rejected": -619.8114013671875, "loss": 0.3495, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -1.5030966997146606, "rewards/margins": 1.5744469165802002, "rewards/rejected": -3.0775437355041504, "step": 2150 }, { "epoch": 0.31797438539673195, "grad_norm": 113.200267427989, "learning_rate": 4.3110536889914996e-07, "logits/chosen": -1.9114919900894165, "logits/rejected": -1.7372190952301025, "logps/chosen": -460.77178955078125, "logps/rejected": -538.3016357421875, "loss": 0.4228, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.6960121393203735, "rewards/margins": 1.0352016687393188, "rewards/rejected": -2.7312138080596924, "step": 2160 }, { "epoch": 0.3194464890328279, "grad_norm": 168.59281094878145, "learning_rate": 4.302172949738626e-07, "logits/chosen": -1.5778995752334595, "logits/rejected": -1.4069231748580933, "logps/chosen": -462.450927734375, "logps/rejected": -779.4312744140625, "loss": 0.3524, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.3175973892211914, "rewards/margins": 2.8890976905822754, "rewards/rejected": -5.206694602966309, "step": 2170 }, { "epoch": 0.3209185926689239, "grad_norm": 278.69400660565157, "learning_rate": 4.2932446127312516e-07, "logits/chosen": -1.5741561651229858, "logits/rejected": -1.5650957822799683, "logps/chosen": -478.7816467285156, "logps/rejected": -707.4798583984375, "loss": 0.5197, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.383969306945801, "rewards/margins": 1.699548363685608, "rewards/rejected": -4.083518028259277, "step": 2180 }, { "epoch": 0.3223906963050199, "grad_norm": 173.69754700155016, "learning_rate": 4.2842689137784825e-07, "logits/chosen": -1.6903053522109985, "logits/rejected": -1.4653757810592651, "logps/chosen": -617.8346557617188, "logps/rejected": -616.084228515625, "loss": 0.6035, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -3.0368387699127197, "rewards/margins": 0.8624863624572754, "rewards/rejected": -3.899325132369995, "step": 2190 }, { "epoch": 0.32386279994111583, "grad_norm": 46.58029651401236, "learning_rate": 4.2752460899403175e-07, "logits/chosen": -1.7009000778198242, "logits/rejected": -1.5140489339828491, "logps/chosen": -391.1360778808594, "logps/rejected": -509.20233154296875, "loss": 0.5781, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.7962887287139893, "rewards/margins": 0.9368594884872437, "rewards/rejected": -2.7331480979919434, "step": 2200 }, { "epoch": 0.32533490357721184, "grad_norm": 81.66337340445386, "learning_rate": 4.2661763795213824e-07, "logits/chosen": -1.603149175643921, "logits/rejected": -1.5082519054412842, "logps/chosen": -343.338623046875, "logps/rejected": -503.3509826660156, "loss": 0.4829, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -1.1309901475906372, "rewards/margins": 1.4072386026382446, "rewards/rejected": -2.538228750228882, "step": 2210 }, { "epoch": 0.3268070072133078, "grad_norm": 99.90977774610141, "learning_rate": 4.2570600220646425e-07, "logits/chosen": -1.8064130544662476, "logits/rejected": -1.7388206720352173, "logps/chosen": -425.31982421875, "logps/rejected": -491.24993896484375, "loss": 0.5276, "rewards/accuracies": 0.5999999642372131, "rewards/chosen": -1.5650538206100464, "rewards/margins": 0.8042703866958618, "rewards/rejected": -2.369324207305908, "step": 2220 }, { "epoch": 0.3282791108494038, "grad_norm": 115.65571685368333, "learning_rate": 4.247897258345071e-07, "logits/chosen": -1.7937599420547485, "logits/rejected": -1.548876166343689, "logps/chosen": -518.07373046875, "logps/rejected": -522.1268310546875, "loss": 0.4908, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.983019232749939, "rewards/margins": 0.8229973912239075, "rewards/rejected": -2.806016445159912, "step": 2230 }, { "epoch": 0.32975121448549977, "grad_norm": 276.4722022286063, "learning_rate": 4.238688330363292e-07, "logits/chosen": -1.5255985260009766, "logits/rejected": -1.411313772201538, "logps/chosen": -432.21160888671875, "logps/rejected": -520.4403076171875, "loss": 0.5167, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -2.198775053024292, "rewards/margins": 1.0241445302963257, "rewards/rejected": -3.222919464111328, "step": 2240 }, { "epoch": 0.3312233181215958, "grad_norm": 108.0234973910894, "learning_rate": 4.2294334813391874e-07, "logits/chosen": -1.619693398475647, "logits/rejected": -1.6066100597381592, "logps/chosen": -473.578125, "logps/rejected": -597.0567016601562, "loss": 0.4687, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.8159301280975342, "rewards/margins": 0.8482751846313477, "rewards/rejected": -2.66420578956604, "step": 2250 }, { "epoch": 0.33269542175769173, "grad_norm": 201.23427831937016, "learning_rate": 4.220132955705476e-07, "logits/chosen": -1.6948457956314087, "logits/rejected": -1.591141939163208, "logps/chosen": -404.19439697265625, "logps/rejected": -464.029052734375, "loss": 0.6185, "rewards/accuracies": 0.7333332300186157, "rewards/chosen": -1.5307927131652832, "rewards/margins": 0.9377376437187195, "rewards/rejected": -2.4685301780700684, "step": 2260 }, { "epoch": 0.33416752539378775, "grad_norm": 70.83276045446581, "learning_rate": 4.2107869991012536e-07, "logits/chosen": -1.7897859811782837, "logits/rejected": -1.5137708187103271, "logps/chosen": -467.86151123046875, "logps/rejected": -523.0823974609375, "loss": 0.4469, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -1.504129409790039, "rewards/margins": 1.249953031539917, "rewards/rejected": -2.754082441329956, "step": 2270 }, { "epoch": 0.3356396290298837, "grad_norm": 83.36314620650249, "learning_rate": 4.201395858365509e-07, "logits/chosen": -1.6233571767807007, "logits/rejected": -1.6392791271209717, "logps/chosen": -397.0567626953125, "logps/rejected": -519.8914794921875, "loss": 0.5112, "rewards/accuracies": 0.76666659116745, "rewards/chosen": -1.665471076965332, "rewards/margins": 0.7333282232284546, "rewards/rejected": -2.398799419403076, "step": 2280 }, { "epoch": 0.33711173266597966, "grad_norm": 90.54641182859237, "learning_rate": 4.191959781530603e-07, "logits/chosen": -1.74801766872406, "logits/rejected": -1.6157410144805908, "logps/chosen": -396.1054382324219, "logps/rejected": -462.4190979003906, "loss": 0.4772, "rewards/accuracies": 0.76666659116745, "rewards/chosen": -1.3713312149047852, "rewards/margins": 1.041359782218933, "rewards/rejected": -2.4126908779144287, "step": 2290 }, { "epoch": 0.33858383630207567, "grad_norm": 275.75693491109797, "learning_rate": 4.1824790178157184e-07, "logits/chosen": -1.643858551979065, "logits/rejected": -1.4274442195892334, "logps/chosen": -501.07159423828125, "logps/rejected": -533.362548828125, "loss": 0.5849, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.1280364990234375, "rewards/margins": 0.8686404228210449, "rewards/rejected": -2.9966769218444824, "step": 2300 }, { "epoch": 0.3400559399381716, "grad_norm": 133.68453914825938, "learning_rate": 4.172953817620275e-07, "logits/chosen": -1.5242453813552856, "logits/rejected": -1.4907231330871582, "logps/chosen": -424.06866455078125, "logps/rejected": -493.7257385253906, "loss": 0.6211, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.9947437047958374, "rewards/margins": 0.778528094291687, "rewards/rejected": -2.7732717990875244, "step": 2310 }, { "epoch": 0.34152804357426764, "grad_norm": 88.6513757240061, "learning_rate": 4.1633844325173215e-07, "logits/chosen": -2.0367541313171387, "logits/rejected": -1.789872407913208, "logps/chosen": -447.4058532714844, "logps/rejected": -527.1895751953125, "loss": 0.448, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.5566468238830566, "rewards/margins": 0.866759181022644, "rewards/rejected": -2.4234061241149902, "step": 2320 }, { "epoch": 0.3430001472103636, "grad_norm": 79.81963281979581, "learning_rate": 4.153771115246886e-07, "logits/chosen": -1.807356595993042, "logits/rejected": -1.7876075506210327, "logps/chosen": -403.1991271972656, "logps/rejected": -438.70452880859375, "loss": 0.5402, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.430990219116211, "rewards/margins": 0.7736988663673401, "rewards/rejected": -2.2046890258789062, "step": 2330 }, { "epoch": 0.3444722508464596, "grad_norm": 82.9727288780879, "learning_rate": 4.144114119709303e-07, "logits/chosen": -1.8029406070709229, "logits/rejected": -1.7416242361068726, "logps/chosen": -419.9112854003906, "logps/rejected": -491.265625, "loss": 0.4891, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.2236841917037964, "rewards/margins": 1.0209158658981323, "rewards/rejected": -2.244600296020508, "step": 2340 }, { "epoch": 0.34594435448255556, "grad_norm": 201.68750219209377, "learning_rate": 4.134413700958509e-07, "logits/chosen": -1.7415510416030884, "logits/rejected": -1.6703112125396729, "logps/chosen": -421.6688537597656, "logps/rejected": -517.2791137695312, "loss": 0.4647, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.4775469303131104, "rewards/margins": 1.1024352312088013, "rewards/rejected": -2.579982280731201, "step": 2350 }, { "epoch": 0.34741645811865157, "grad_norm": 170.76824942269258, "learning_rate": 4.1246701151953014e-07, "logits/chosen": -1.648890495300293, "logits/rejected": -1.5278441905975342, "logps/chosen": -426.95556640625, "logps/rejected": -452.89178466796875, "loss": 0.5879, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.9528697729110718, "rewards/margins": 0.5463830232620239, "rewards/rejected": -2.499253273010254, "step": 2360 }, { "epoch": 0.34888856175474753, "grad_norm": 152.41564563619943, "learning_rate": 4.1148836197605774e-07, "logits/chosen": -1.845446228981018, "logits/rejected": -1.8086979389190674, "logps/chosen": -420.7265625, "logps/rejected": -522.8754272460938, "loss": 0.5232, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.6788885593414307, "rewards/margins": 1.1555010080337524, "rewards/rejected": -2.8343894481658936, "step": 2370 }, { "epoch": 0.35036066539084354, "grad_norm": 99.5333377976858, "learning_rate": 4.105054473128536e-07, "logits/chosen": -1.5415117740631104, "logits/rejected": -1.571418046951294, "logps/chosen": -339.0941162109375, "logps/rejected": -409.9202575683594, "loss": 0.5038, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.386282205581665, "rewards/margins": 0.786494255065918, "rewards/rejected": -2.172776699066162, "step": 2380 }, { "epoch": 0.3518327690269395, "grad_norm": 71.6492219061604, "learning_rate": 4.0951829348998477e-07, "logits/chosen": -1.7694034576416016, "logits/rejected": -1.6417080163955688, "logps/chosen": -485.8193359375, "logps/rejected": -523.8450927734375, "loss": 0.5399, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.9029020071029663, "rewards/margins": 0.8837946057319641, "rewards/rejected": -2.7866969108581543, "step": 2390 }, { "epoch": 0.35330487266303545, "grad_norm": 106.60260065713516, "learning_rate": 4.0852692657948027e-07, "logits/chosen": -1.729047417640686, "logits/rejected": -1.5688316822052002, "logps/chosen": -428.2808532714844, "logps/rejected": -506.84942626953125, "loss": 0.4931, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -2.001753091812134, "rewards/margins": 0.9478896260261536, "rewards/rejected": -2.9496426582336426, "step": 2400 }, { "epoch": 0.35477697629913146, "grad_norm": 165.64069155776042, "learning_rate": 4.075313727646422e-07, "logits/chosen": -1.7659275531768799, "logits/rejected": -1.659751296043396, "logps/chosen": -427.1341857910156, "logps/rejected": -658.7684326171875, "loss": 0.4037, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -1.53098464012146, "rewards/margins": 2.267085075378418, "rewards/rejected": -3.798069715499878, "step": 2410 }, { "epoch": 0.3562490799352274, "grad_norm": 80.33914214981517, "learning_rate": 4.0653165833935433e-07, "logits/chosen": -1.7472379207611084, "logits/rejected": -1.6408731937408447, "logps/chosen": -430.6123046875, "logps/rejected": -450.4761657714844, "loss": 0.4999, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.6223323345184326, "rewards/margins": 0.9185541272163391, "rewards/rejected": -2.540886640548706, "step": 2420 }, { "epoch": 0.35772118357132343, "grad_norm": 155.16471959016783, "learning_rate": 4.0552780970738755e-07, "logits/chosen": -1.6454254388809204, "logits/rejected": -1.722833275794983, "logps/chosen": -406.28668212890625, "logps/rejected": -606.0823364257812, "loss": 0.497, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.685489296913147, "rewards/margins": 1.4034935235977173, "rewards/rejected": -3.0889828205108643, "step": 2430 }, { "epoch": 0.3591932872074194, "grad_norm": 77.22939282764068, "learning_rate": 4.045198533817028e-07, "logits/chosen": -1.6065928936004639, "logits/rejected": -1.5150026082992554, "logps/chosen": -460.1014709472656, "logps/rejected": -479.58551025390625, "loss": 0.5253, "rewards/accuracies": 0.5666666626930237, "rewards/chosen": -2.1393537521362305, "rewards/margins": 0.4858306050300598, "rewards/rejected": -2.6251842975616455, "step": 2440 }, { "epoch": 0.3606653908435154, "grad_norm": 72.37032769518197, "learning_rate": 4.0350781598375027e-07, "logits/chosen": -1.8797813653945923, "logits/rejected": -1.5134384632110596, "logps/chosen": -555.326416015625, "logps/rejected": -551.3318481445312, "loss": 0.5071, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.8028364181518555, "rewards/margins": 1.157103180885315, "rewards/rejected": -2.95993971824646, "step": 2450 }, { "epoch": 0.36213749447961135, "grad_norm": 44.34521151604621, "learning_rate": 4.024917242427669e-07, "logits/chosen": -1.761695146560669, "logits/rejected": -1.6549463272094727, "logps/chosen": -493.4774475097656, "logps/rejected": -565.1205444335938, "loss": 0.5217, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -2.094701051712036, "rewards/margins": 0.9296547770500183, "rewards/rejected": -3.02435564994812, "step": 2460 }, { "epoch": 0.36360959811570737, "grad_norm": 131.94375909507465, "learning_rate": 4.0147160499507006e-07, "logits/chosen": -1.7216295003890991, "logits/rejected": -1.5743701457977295, "logps/chosen": -556.4461059570312, "logps/rejected": -554.2019653320312, "loss": 0.5745, "rewards/accuracies": 0.5666666626930237, "rewards/chosen": -2.2372331619262695, "rewards/margins": 0.646490752696991, "rewards/rejected": -2.883723735809326, "step": 2470 }, { "epoch": 0.3650817017518033, "grad_norm": 86.02155564362292, "learning_rate": 4.004474851833488e-07, "logits/chosen": -1.9678194522857666, "logits/rejected": -1.798370361328125, "logps/chosen": -569.799560546875, "logps/rejected": -543.369140625, "loss": 0.4965, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.6525856256484985, "rewards/margins": 0.9285826683044434, "rewards/rejected": -2.5811684131622314, "step": 2480 }, { "epoch": 0.36655380538789933, "grad_norm": 69.154632954316, "learning_rate": 3.9941939185595255e-07, "logits/chosen": -1.6970447301864624, "logits/rejected": -1.6858055591583252, "logps/chosen": -420.1490783691406, "logps/rejected": -516.6390380859375, "loss": 0.5071, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.7808294296264648, "rewards/margins": 0.8183453679084778, "rewards/rejected": -2.599174976348877, "step": 2490 }, { "epoch": 0.3680259090239953, "grad_norm": 51.65539198848958, "learning_rate": 3.9838735216617615e-07, "logits/chosen": -1.9549919366836548, "logits/rejected": -1.7374999523162842, "logps/chosen": -404.2395324707031, "logps/rejected": -411.1822204589844, "loss": 0.5923, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.5497435331344604, "rewards/margins": 0.46222543716430664, "rewards/rejected": -2.0119690895080566, "step": 2500 }, { "epoch": 0.36949801266009125, "grad_norm": 108.0607252501813, "learning_rate": 3.9735139337154334e-07, "logits/chosen": -1.6869624853134155, "logits/rejected": -1.535576343536377, "logps/chosen": -372.4644775390625, "logps/rejected": -396.6224365234375, "loss": 0.7073, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.7658885717391968, "rewards/margins": 0.41729116439819336, "rewards/rejected": -2.1831798553466797, "step": 2510 }, { "epoch": 0.37097011629618726, "grad_norm": 205.528312081728, "learning_rate": 3.963115428330864e-07, "logits/chosen": -1.9358739852905273, "logits/rejected": -1.8595073223114014, "logps/chosen": -477.067138671875, "logps/rejected": -510.2032165527344, "loss": 0.5285, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.340234398841858, "rewards/margins": 0.4346332550048828, "rewards/rejected": -1.7748676538467407, "step": 2520 }, { "epoch": 0.3724422199322832, "grad_norm": 62.93162090010597, "learning_rate": 3.9526782801462384e-07, "logits/chosen": -1.9853556156158447, "logits/rejected": -1.5232021808624268, "logps/chosen": -439.63507080078125, "logps/rejected": -455.38641357421875, "loss": 0.4636, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -1.28523850440979, "rewards/margins": 1.3732984066009521, "rewards/rejected": -2.658536672592163, "step": 2530 }, { "epoch": 0.3739143235683792, "grad_norm": 92.80607468315416, "learning_rate": 3.9422027648203474e-07, "logits/chosen": -1.4608604907989502, "logits/rejected": -1.4058932065963745, "logps/chosen": -407.0013122558594, "logps/rejected": -442.52569580078125, "loss": 0.5356, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.1296303272247314, "rewards/margins": 0.35629209876060486, "rewards/rejected": -2.485922336578369, "step": 2540 }, { "epoch": 0.3753864272044752, "grad_norm": 79.96693950549795, "learning_rate": 3.9316891590253086e-07, "logits/chosen": -1.9654786586761475, "logits/rejected": -1.748862862586975, "logps/chosen": -483.31915283203125, "logps/rejected": -529.26025390625, "loss": 0.5462, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.003206729888916, "rewards/margins": 0.8893306851387024, "rewards/rejected": -2.8925373554229736, "step": 2550 }, { "epoch": 0.3768585308405712, "grad_norm": 56.42140357987996, "learning_rate": 3.921137740439261e-07, "logits/chosen": -1.9938732385635376, "logits/rejected": -1.8925504684448242, "logps/chosen": -409.49664306640625, "logps/rejected": -457.82708740234375, "loss": 0.4268, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.618497610092163, "rewards/margins": 0.8208115696907043, "rewards/rejected": -2.4393093585968018, "step": 2560 }, { "epoch": 0.37833063447666715, "grad_norm": 66.24397541381366, "learning_rate": 3.910548787739024e-07, "logits/chosen": -2.0649988651275635, "logits/rejected": -1.8993663787841797, "logps/chosen": -502.3565368652344, "logps/rejected": -589.432373046875, "loss": 0.5242, "rewards/accuracies": 0.73333340883255, "rewards/chosen": -2.220254421234131, "rewards/margins": 0.9584907293319702, "rewards/rejected": -3.1787452697753906, "step": 2570 }, { "epoch": 0.37980273811276316, "grad_norm": 88.89202049332125, "learning_rate": 3.8999225805927483e-07, "logits/chosen": -1.9994415044784546, "logits/rejected": -1.6600345373153687, "logps/chosen": -461.4424743652344, "logps/rejected": -531.6176147460938, "loss": 0.5575, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -2.0529115200042725, "rewards/margins": 0.8937948942184448, "rewards/rejected": -2.9467062950134277, "step": 2580 }, { "epoch": 0.3812748417488591, "grad_norm": 57.87439072258639, "learning_rate": 3.889259399652519e-07, "logits/chosen": -1.8909122943878174, "logits/rejected": -1.6752264499664307, "logps/chosen": -532.424560546875, "logps/rejected": -600.0192260742188, "loss": 0.4089, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -2.1988584995269775, "rewards/margins": 1.2118005752563477, "rewards/rejected": -3.410658597946167, "step": 2590 }, { "epoch": 0.38274694538495513, "grad_norm": 187.69946919712814, "learning_rate": 3.8785595265469497e-07, "logits/chosen": -1.657313346862793, "logits/rejected": -1.617428183555603, "logps/chosen": -478.59722900390625, "logps/rejected": -561.6739501953125, "loss": 0.4847, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.2130188941955566, "rewards/margins": 1.131255865097046, "rewards/rejected": -3.3442745208740234, "step": 2600 }, { "epoch": 0.3842190490210511, "grad_norm": 126.62049634548895, "learning_rate": 3.867823243873743e-07, "logits/chosen": -1.6318082809448242, "logits/rejected": -1.4904166460037231, "logps/chosen": -422.03607177734375, "logps/rejected": -512.8194580078125, "loss": 0.414, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -2.04823637008667, "rewards/margins": 1.1126116514205933, "rewards/rejected": -3.1608481407165527, "step": 2610 }, { "epoch": 0.38569115265714704, "grad_norm": 84.95647857185706, "learning_rate": 3.8570508351922234e-07, "logits/chosen": -1.6178743839263916, "logits/rejected": -1.45248544216156, "logps/chosen": -510.5176696777344, "logps/rejected": -627.6212158203125, "loss": 0.4889, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -2.243734836578369, "rewards/margins": 1.4953174591064453, "rewards/rejected": -3.7390522956848145, "step": 2620 }, { "epoch": 0.38716325629324305, "grad_norm": 217.3610849910934, "learning_rate": 3.8462425850158533e-07, "logits/chosen": -1.5772051811218262, "logits/rejected": -1.6962175369262695, "logps/chosen": -454.8011169433594, "logps/rejected": -596.3443603515625, "loss": 0.5157, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -2.3009824752807617, "rewards/margins": 1.1165456771850586, "rewards/rejected": -3.4175286293029785, "step": 2630 }, { "epoch": 0.388635359929339, "grad_norm": 93.71271391232355, "learning_rate": 3.8353987788047136e-07, "logits/chosen": -1.7966476678848267, "logits/rejected": -1.6472831964492798, "logps/chosen": -525.8533935546875, "logps/rejected": -578.4793090820312, "loss": 0.4526, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -2.2422289848327637, "rewards/margins": 0.9489534497261047, "rewards/rejected": -3.1911826133728027, "step": 2640 }, { "epoch": 0.390107463565435, "grad_norm": 117.14755197392523, "learning_rate": 3.824519702957969e-07, "logits/chosen": -1.8352773189544678, "logits/rejected": -1.7037101984024048, "logps/chosen": -601.3973999023438, "logps/rejected": -587.599609375, "loss": 0.7123, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -2.5357017517089844, "rewards/margins": 0.8538087606430054, "rewards/rejected": -3.3895103931427, "step": 2650 }, { "epoch": 0.391579567201531, "grad_norm": 100.67283128846917, "learning_rate": 3.8136056448063016e-07, "logits/chosen": -1.969740867614746, "logits/rejected": -1.8629324436187744, "logps/chosen": -484.20343017578125, "logps/rejected": -623.115966796875, "loss": 0.5587, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -2.134702682495117, "rewards/margins": 1.5543760061264038, "rewards/rejected": -3.6890788078308105, "step": 2660 }, { "epoch": 0.393051670837627, "grad_norm": 68.67861913584042, "learning_rate": 3.802656892604319e-07, "logits/chosen": -1.7493486404418945, "logits/rejected": -1.6327745914459229, "logps/chosen": -416.34039306640625, "logps/rejected": -526.93603515625, "loss": 0.5177, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.926028847694397, "rewards/margins": 1.4044030904769897, "rewards/rejected": -3.330432176589966, "step": 2670 }, { "epoch": 0.39452377447372294, "grad_norm": 83.42034644974696, "learning_rate": 3.791673735522949e-07, "logits/chosen": -1.7801506519317627, "logits/rejected": -1.6689989566802979, "logps/chosen": -450.91168212890625, "logps/rejected": -604.1636962890625, "loss": 0.5419, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.969585657119751, "rewards/margins": 1.1824830770492554, "rewards/rejected": -3.152069091796875, "step": 2680 }, { "epoch": 0.39599587810981896, "grad_norm": 92.00474092190898, "learning_rate": 3.7806564636417936e-07, "logits/chosen": -1.8807475566864014, "logits/rejected": -1.7490568161010742, "logps/chosen": -384.54510498046875, "logps/rejected": -480.938232421875, "loss": 0.5335, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -1.2384569644927979, "rewards/margins": 1.1380057334899902, "rewards/rejected": -2.376462936401367, "step": 2690 }, { "epoch": 0.3974679817459149, "grad_norm": 61.0433259916579, "learning_rate": 3.769605367941472e-07, "logits/chosen": -1.7871767282485962, "logits/rejected": -1.7239713668823242, "logps/chosen": -387.7853698730469, "logps/rejected": -457.2977600097656, "loss": 0.43, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.350097894668579, "rewards/margins": 0.8648613095283508, "rewards/rejected": -2.214958906173706, "step": 2700 }, { "epoch": 0.39894008538201087, "grad_norm": 91.02215737867614, "learning_rate": 3.7585207402959377e-07, "logits/chosen": -1.882582426071167, "logits/rejected": -1.7801997661590576, "logps/chosen": -335.1067199707031, "logps/rejected": -406.0661315917969, "loss": 0.5354, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -0.9968665838241577, "rewards/margins": 0.7382841110229492, "rewards/rejected": -1.735150694847107, "step": 2710 }, { "epoch": 0.4004121890181069, "grad_norm": 39.49209571031024, "learning_rate": 3.747402873464764e-07, "logits/chosen": -1.8460311889648438, "logits/rejected": -1.5826148986816406, "logps/chosen": -379.17303466796875, "logps/rejected": -449.3412170410156, "loss": 0.4803, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.0963877439498901, "rewards/margins": 1.0909336805343628, "rewards/rejected": -2.187321424484253, "step": 2720 }, { "epoch": 0.40188429265420283, "grad_norm": 499.5492917203205, "learning_rate": 3.7362520610854147e-07, "logits/chosen": -1.8000824451446533, "logits/rejected": -1.6837491989135742, "logps/chosen": -302.5549621582031, "logps/rejected": -401.45684814453125, "loss": 0.539, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -0.9969806671142578, "rewards/margins": 0.9101873636245728, "rewards/rejected": -1.9071677923202515, "step": 2730 }, { "epoch": 0.40335639629029885, "grad_norm": 189.95202996672072, "learning_rate": 3.725068597665491e-07, "logits/chosen": -1.6278194189071655, "logits/rejected": -1.5598769187927246, "logps/chosen": -362.18670654296875, "logps/rejected": -529.1080322265625, "loss": 0.5633, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -1.668229103088379, "rewards/margins": 0.9493568539619446, "rewards/rejected": -2.617586135864258, "step": 2740 }, { "epoch": 0.4048284999263948, "grad_norm": 57.17856940250938, "learning_rate": 3.71385277857495e-07, "logits/chosen": -1.5626249313354492, "logits/rejected": -1.5322986841201782, "logps/chosen": -467.0201721191406, "logps/rejected": -592.6168823242188, "loss": 0.6194, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.0915586948394775, "rewards/margins": 1.053040862083435, "rewards/rejected": -3.144599676132202, "step": 2750 }, { "epoch": 0.4063006035624908, "grad_norm": 123.65770816088971, "learning_rate": 3.702604900038302e-07, "logits/chosen": -1.8136831521987915, "logits/rejected": -1.6411972045898438, "logps/chosen": -401.2938537597656, "logps/rejected": -481.94866943359375, "loss": 0.5359, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.6743227243423462, "rewards/margins": 1.1585530042648315, "rewards/rejected": -2.8328757286071777, "step": 2760 }, { "epoch": 0.40777270719858677, "grad_norm": 143.9870781335657, "learning_rate": 3.691325259126794e-07, "logits/chosen": -1.6237173080444336, "logits/rejected": -1.6452341079711914, "logps/chosen": -437.26226806640625, "logps/rejected": -552.0536499023438, "loss": 0.5759, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.1095197200775146, "rewards/margins": 0.8526825904846191, "rewards/rejected": -2.962202310562134, "step": 2770 }, { "epoch": 0.4092448108346828, "grad_norm": 180.00670354693986, "learning_rate": 3.6800141537505556e-07, "logits/chosen": -1.767045259475708, "logits/rejected": -1.6805870532989502, "logps/chosen": -439.8860778808594, "logps/rejected": -606.5364379882812, "loss": 0.4544, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -2.072852373123169, "rewards/margins": 1.1825932264328003, "rewards/rejected": -3.2554454803466797, "step": 2780 }, { "epoch": 0.41071691447077874, "grad_norm": 200.81887663430837, "learning_rate": 3.6686718826507363e-07, "logits/chosen": -1.7911945581436157, "logits/rejected": -1.6554672718048096, "logps/chosen": -517.91162109375, "logps/rejected": -578.0740356445312, "loss": 0.663, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -2.175565004348755, "rewards/margins": 0.9195197820663452, "rewards/rejected": -3.0950846672058105, "step": 2790 }, { "epoch": 0.41218901810687475, "grad_norm": 51.678066494413, "learning_rate": 3.6572987453916114e-07, "logits/chosen": -1.5547738075256348, "logits/rejected": -1.453906774520874, "logps/chosen": -441.12518310546875, "logps/rejected": -508.16717529296875, "loss": 0.4146, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.0770745277404785, "rewards/margins": 1.0565335750579834, "rewards/rejected": -3.133607864379883, "step": 2800 }, { "epoch": 0.4136611217429707, "grad_norm": 100.40668329912171, "learning_rate": 3.645895042352672e-07, "logits/chosen": -1.7121965885162354, "logits/rejected": -1.528241753578186, "logps/chosen": -455.609130859375, "logps/rejected": -613.9754028320312, "loss": 0.4391, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.7937233448028564, "rewards/margins": 1.7376922369003296, "rewards/rejected": -3.5314154624938965, "step": 2810 }, { "epoch": 0.41513322537906666, "grad_norm": 214.75386932867553, "learning_rate": 3.634461074720695e-07, "logits/chosen": -1.533944845199585, "logits/rejected": -1.544070839881897, "logps/chosen": -434.95330810546875, "logps/rejected": -532.8826904296875, "loss": 0.5883, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.9687788486480713, "rewards/margins": 1.018162488937378, "rewards/rejected": -2.986941337585449, "step": 2820 }, { "epoch": 0.4166053290151627, "grad_norm": 128.46106527602066, "learning_rate": 3.622997144481781e-07, "logits/chosen": -1.6738640069961548, "logits/rejected": -1.3706481456756592, "logps/chosen": -503.4684143066406, "logps/rejected": -481.333740234375, "loss": 0.5221, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.184661388397217, "rewards/margins": 0.5752471089363098, "rewards/rejected": -2.759908437728882, "step": 2830 }, { "epoch": 0.41807743265125863, "grad_norm": 65.25789469781631, "learning_rate": 3.611503554413383e-07, "logits/chosen": -1.8849303722381592, "logits/rejected": -1.672583818435669, "logps/chosen": -434.37841796875, "logps/rejected": -500.3805236816406, "loss": 0.5826, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.4656105041503906, "rewards/margins": 0.8493620157241821, "rewards/rejected": -2.314972400665283, "step": 2840 }, { "epoch": 0.41954953628735464, "grad_norm": 133.59046261362076, "learning_rate": 3.599980608076312e-07, "logits/chosen": -1.7649234533309937, "logits/rejected": -1.5637601613998413, "logps/chosen": -408.22589111328125, "logps/rejected": -450.2254943847656, "loss": 0.5604, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.1920119524002075, "rewards/margins": 1.0949840545654297, "rewards/rejected": -2.2869958877563477, "step": 2850 }, { "epoch": 0.4210216399234506, "grad_norm": 104.90643999171841, "learning_rate": 3.5884286098067124e-07, "logits/chosen": -1.9010881185531616, "logits/rejected": -1.6664737462997437, "logps/chosen": -481.55255126953125, "logps/rejected": -533.0050659179688, "loss": 0.467, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.788642168045044, "rewards/margins": 0.8375269174575806, "rewards/rejected": -2.626168727874756, "step": 2860 }, { "epoch": 0.4224937435595466, "grad_norm": 127.93392264394856, "learning_rate": 3.5768478647080315e-07, "logits/chosen": -1.5535461902618408, "logits/rejected": -1.6039278507232666, "logps/chosen": -457.58758544921875, "logps/rejected": -516.9219970703125, "loss": 0.536, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.6912761926651, "rewards/margins": 0.9885860681533813, "rewards/rejected": -2.6798622608184814, "step": 2870 }, { "epoch": 0.42396584719564256, "grad_norm": 76.66951905944022, "learning_rate": 3.565238678642957e-07, "logits/chosen": -1.4828526973724365, "logits/rejected": -1.0871570110321045, "logps/chosen": -514.9000244140625, "logps/rejected": -565.3880615234375, "loss": 0.479, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -2.057191848754883, "rewards/margins": 1.4782410860061646, "rewards/rejected": -3.535432815551758, "step": 2880 }, { "epoch": 0.4254379508317386, "grad_norm": 74.1073437013685, "learning_rate": 3.55360135822534e-07, "logits/chosen": -1.6502548456192017, "logits/rejected": -1.4351156949996948, "logps/chosen": -477.233642578125, "logps/rejected": -571.50732421875, "loss": 0.4825, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.9168500900268555, "rewards/margins": 1.1503159999847412, "rewards/rejected": -3.0671660900115967, "step": 2890 }, { "epoch": 0.42691005446783453, "grad_norm": 126.75011331732391, "learning_rate": 3.541936210812095e-07, "logits/chosen": -1.5748753547668457, "logits/rejected": -1.2683310508728027, "logps/chosen": -476.5843200683594, "logps/rejected": -569.3049926757812, "loss": 0.3947, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -2.0876541137695312, "rewards/margins": 1.4470094442367554, "rewards/rejected": -3.534663438796997, "step": 2900 }, { "epoch": 0.42838215810393054, "grad_norm": 76.34029595140632, "learning_rate": 3.5302435444950894e-07, "logits/chosen": -1.5367224216461182, "logits/rejected": -1.1965324878692627, "logps/chosen": -472.9261779785156, "logps/rejected": -614.3412475585938, "loss": 0.6052, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.8904402256011963, "rewards/margins": 1.9703643321990967, "rewards/rejected": -3.860804319381714, "step": 2910 }, { "epoch": 0.4298542617400265, "grad_norm": 99.61793313483508, "learning_rate": 3.518523668092994e-07, "logits/chosen": -1.2857729196548462, "logits/rejected": -1.0071913003921509, "logps/chosen": -548.1761474609375, "logps/rejected": -625.6806030273438, "loss": 0.5609, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.728156566619873, "rewards/margins": 1.225595235824585, "rewards/rejected": -3.9537525177001953, "step": 2920 }, { "epoch": 0.43132636537612246, "grad_norm": 88.6332661953972, "learning_rate": 3.506776891143138e-07, "logits/chosen": -1.552356481552124, "logits/rejected": -1.6167726516723633, "logps/chosen": -448.96484375, "logps/rejected": -624.7279052734375, "loss": 0.5411, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.9730370044708252, "rewards/margins": 1.0921411514282227, "rewards/rejected": -3.065178155899048, "step": 2930 }, { "epoch": 0.43279846901221847, "grad_norm": 83.40511332062242, "learning_rate": 3.495003523893329e-07, "logits/chosen": -1.5380442142486572, "logits/rejected": -1.5943825244903564, "logps/chosen": -439.9825134277344, "logps/rejected": -508.53851318359375, "loss": 0.5666, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -2.0246939659118652, "rewards/margins": 0.7639306783676147, "rewards/rejected": -2.7886245250701904, "step": 2940 }, { "epoch": 0.4342705726483144, "grad_norm": 55.51987780128476, "learning_rate": 3.4832038772936574e-07, "logits/chosen": -1.8873517513275146, "logits/rejected": -1.7539236545562744, "logps/chosen": -405.72906494140625, "logps/rejected": -465.75433349609375, "loss": 0.5226, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.5367895364761353, "rewards/margins": 0.5773477554321289, "rewards/rejected": -2.1141371726989746, "step": 2950 }, { "epoch": 0.43574267628441044, "grad_norm": 93.65796111677007, "learning_rate": 3.471378262988288e-07, "logits/chosen": -1.706993818283081, "logits/rejected": -1.5690408945083618, "logps/chosen": -467.59619140625, "logps/rejected": -549.7493896484375, "loss": 0.4985, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.867831826210022, "rewards/margins": 1.3700368404388428, "rewards/rejected": -3.237868547439575, "step": 2960 }, { "epoch": 0.4372147799205064, "grad_norm": 96.3640973140334, "learning_rate": 3.459526993307225e-07, "logits/chosen": -1.6291742324829102, "logits/rejected": -1.3714163303375244, "logps/chosen": -563.1551513671875, "logps/rejected": -566.9429931640625, "loss": 0.6511, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.3630969524383545, "rewards/margins": 0.7266470789909363, "rewards/rejected": -3.0897438526153564, "step": 2970 }, { "epoch": 0.4386868835566024, "grad_norm": 90.42039043244793, "learning_rate": 3.4476503812580677e-07, "logits/chosen": -1.7459440231323242, "logits/rejected": -1.4759334325790405, "logps/chosen": -484.9942321777344, "logps/rejected": -576.6343383789062, "loss": 0.5027, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -1.9247719049453735, "rewards/margins": 1.0805116891860962, "rewards/rejected": -3.0052835941314697, "step": 2980 }, { "epoch": 0.44015898719269836, "grad_norm": 76.09557851880464, "learning_rate": 3.4357487405177367e-07, "logits/chosen": -1.689424753189087, "logits/rejected": -1.6925474405288696, "logps/chosen": -386.05340576171875, "logps/rejected": -513.0780029296875, "loss": 0.4676, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.5728152990341187, "rewards/margins": 1.129425048828125, "rewards/rejected": -2.702240228652954, "step": 2990 }, { "epoch": 0.44163109082879437, "grad_norm": 93.808001505817, "learning_rate": 3.423822385424195e-07, "logits/chosen": -1.7472833395004272, "logits/rejected": -1.7409921884536743, "logps/chosen": -445.745849609375, "logps/rejected": -571.5545043945312, "loss": 0.4092, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.7007602453231812, "rewards/margins": 1.3439972400665283, "rewards/rejected": -3.04475736618042, "step": 3000 }, { "epoch": 0.4431031944648903, "grad_norm": 61.08186229678611, "learning_rate": 3.411871630968145e-07, "logits/chosen": -1.5771856307983398, "logits/rejected": -1.3157498836517334, "logps/chosen": -528.1099853515625, "logps/rejected": -744.0299072265625, "loss": 0.4846, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -2.2174181938171387, "rewards/margins": 2.1569695472717285, "rewards/rejected": -4.374387264251709, "step": 3010 }, { "epoch": 0.4445752981009863, "grad_norm": 152.75349645859527, "learning_rate": 3.3998967927847067e-07, "logits/chosen": -1.6811447143554688, "logits/rejected": -1.5367097854614258, "logps/chosen": -531.6951904296875, "logps/rejected": -636.5330200195312, "loss": 0.5657, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -1.935083031654358, "rewards/margins": 1.2397915124893188, "rewards/rejected": -3.1748743057250977, "step": 3020 }, { "epoch": 0.4460474017370823, "grad_norm": 85.31949008343528, "learning_rate": 3.3878981871450846e-07, "logits/chosen": -1.7278846502304077, "logits/rejected": -1.5852535963058472, "logps/chosen": -497.8582458496094, "logps/rejected": -600.8698120117188, "loss": 0.5145, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.7871097326278687, "rewards/margins": 0.9624664187431335, "rewards/rejected": -2.7495760917663574, "step": 3030 }, { "epoch": 0.44751950537317825, "grad_norm": 124.42465815879098, "learning_rate": 3.375876130948211e-07, "logits/chosen": -1.554373025894165, "logits/rejected": -1.2637821435928345, "logps/chosen": -499.805908203125, "logps/rejected": -618.772705078125, "loss": 0.4519, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -2.3066258430480957, "rewards/margins": 1.3394854068756104, "rewards/rejected": -3.6461117267608643, "step": 3040 }, { "epoch": 0.44899160900927426, "grad_norm": 123.63696424661978, "learning_rate": 3.3638309417123824e-07, "logits/chosen": -1.5253585577011108, "logits/rejected": -1.2020318508148193, "logps/chosen": -526.9351806640625, "logps/rejected": -545.9700927734375, "loss": 0.5424, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -2.3755273818969727, "rewards/margins": 0.7981761693954468, "rewards/rejected": -3.173703670501709, "step": 3050 }, { "epoch": 0.4504637126453702, "grad_norm": 75.00017482503124, "learning_rate": 3.3517629375668615e-07, "logits/chosen": -1.6559574604034424, "logits/rejected": -1.4047844409942627, "logps/chosen": -521.6126098632812, "logps/rejected": -628.2869873046875, "loss": 0.4713, "rewards/accuracies": 0.7333332300186157, "rewards/chosen": -2.304652452468872, "rewards/margins": 1.2449811697006226, "rewards/rejected": -3.549633502960205, "step": 3060 }, { "epoch": 0.45193581628146623, "grad_norm": 87.53956831154406, "learning_rate": 3.3396724372434914e-07, "logits/chosen": -1.7649757862091064, "logits/rejected": -1.41666579246521, "logps/chosen": -513.6778564453125, "logps/rejected": -516.7106323242188, "loss": 0.502, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.6286847591400146, "rewards/margins": 1.3104242086410522, "rewards/rejected": -2.9391090869903564, "step": 3070 }, { "epoch": 0.4534079199175622, "grad_norm": 96.42195669627209, "learning_rate": 3.327559760068263e-07, "logits/chosen": -1.2842210531234741, "logits/rejected": -1.3093006610870361, "logps/chosen": -361.3973693847656, "logps/rejected": -591.5134887695312, "loss": 0.5441, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.7170593738555908, "rewards/margins": 1.7197749614715576, "rewards/rejected": -3.4368338584899902, "step": 3080 }, { "epoch": 0.4548800235536582, "grad_norm": 125.78559949794136, "learning_rate": 3.3154252259528883e-07, "logits/chosen": -1.7974132299423218, "logits/rejected": -1.365867257118225, "logps/chosen": -489.73651123046875, "logps/rejected": -562.1240234375, "loss": 0.4368, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.535992980003357, "rewards/margins": 1.3945261240005493, "rewards/rejected": -2.930518865585327, "step": 3090 }, { "epoch": 0.45635212718975415, "grad_norm": 132.73671307201735, "learning_rate": 3.303269155386351e-07, "logits/chosen": -1.588967204093933, "logits/rejected": -1.541374921798706, "logps/chosen": -450.5350036621094, "logps/rejected": -591.4512329101562, "loss": 0.515, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.714499831199646, "rewards/margins": 1.3883813619613647, "rewards/rejected": -3.1028809547424316, "step": 3100 }, { "epoch": 0.45782423082585016, "grad_norm": 84.35226676195592, "learning_rate": 3.291091869426439e-07, "logits/chosen": -1.4360454082489014, "logits/rejected": -1.4084079265594482, "logps/chosen": -489.7723693847656, "logps/rejected": -656.9114990234375, "loss": 0.4784, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -1.9550676345825195, "rewards/margins": 2.1726884841918945, "rewards/rejected": -4.127756118774414, "step": 3110 }, { "epoch": 0.4592963344619461, "grad_norm": 138.7719561127843, "learning_rate": 3.27889368969127e-07, "logits/chosen": -0.8632322549819946, "logits/rejected": -0.5557774305343628, "logps/chosen": -555.629150390625, "logps/rejected": -620.8408813476562, "loss": 0.4959, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -2.7310373783111572, "rewards/margins": 1.4438550472259521, "rewards/rejected": -4.174892425537109, "step": 3120 }, { "epoch": 0.4607684380980421, "grad_norm": 86.88693560595163, "learning_rate": 3.266674938350789e-07, "logits/chosen": -0.8334887623786926, "logits/rejected": -0.2807777523994446, "logps/chosen": -621.770751953125, "logps/rejected": -709.0192260742188, "loss": 0.4996, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -3.182051181793213, "rewards/margins": 1.3985031843185425, "rewards/rejected": -4.580554485321045, "step": 3130 }, { "epoch": 0.4622405417341381, "grad_norm": 101.13490739996732, "learning_rate": 3.254435938118269e-07, "logits/chosen": -0.32349011301994324, "logits/rejected": -1.0005395412445068, "logps/chosen": -595.7144165039062, "logps/rejected": -687.2969970703125, "loss": 0.5751, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -2.945071220397949, "rewards/margins": 0.8556967973709106, "rewards/rejected": -3.8007683753967285, "step": 3140 }, { "epoch": 0.46371264537023404, "grad_norm": 112.52931558596427, "learning_rate": 3.242177012241778e-07, "logits/chosen": -0.6440261602401733, "logits/rejected": 0.2714475989341736, "logps/chosen": -611.9390258789062, "logps/rejected": -653.388916015625, "loss": 0.5795, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -3.470188856124878, "rewards/margins": 1.012743592262268, "rewards/rejected": -4.482932090759277, "step": 3150 }, { "epoch": 0.46518474900633006, "grad_norm": 98.97133750890707, "learning_rate": 3.229898484495649e-07, "logits/chosen": -0.5198215246200562, "logits/rejected": -0.6536621451377869, "logps/chosen": -457.37518310546875, "logps/rejected": -566.5096435546875, "loss": 0.5241, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -2.681630849838257, "rewards/margins": 0.9847855567932129, "rewards/rejected": -3.666416883468628, "step": 3160 }, { "epoch": 0.466656852642426, "grad_norm": 74.33658755806334, "learning_rate": 3.2176006791719266e-07, "logits/chosen": -0.8453855514526367, "logits/rejected": -0.7567756772041321, "logps/chosen": -488.2428283691406, "logps/rejected": -611.7319946289062, "loss": 0.4892, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.6312668323516846, "rewards/margins": 1.1969882249832153, "rewards/rejected": -3.8282551765441895, "step": 3170 }, { "epoch": 0.468128956278522, "grad_norm": 172.69730178456533, "learning_rate": 3.2052839210718007e-07, "logits/chosen": -0.9820365905761719, "logits/rejected": -1.2467740774154663, "logps/chosen": -452.60357666015625, "logps/rejected": -584.418212890625, "loss": 0.4949, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -2.2015039920806885, "rewards/margins": 0.8768129348754883, "rewards/rejected": -3.078317165374756, "step": 3180 }, { "epoch": 0.469601059914618, "grad_norm": 63.62100677922358, "learning_rate": 3.1929485354970297e-07, "logits/chosen": -1.4609642028808594, "logits/rejected": -1.1372168064117432, "logps/chosen": -481.2571716308594, "logps/rejected": -655.3583374023438, "loss": 0.3664, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.9624278545379639, "rewards/margins": 1.7153165340423584, "rewards/rejected": -3.677743911743164, "step": 3190 }, { "epoch": 0.471073163550714, "grad_norm": 60.70618218289404, "learning_rate": 3.180594848241346e-07, "logits/chosen": -1.1690118312835693, "logits/rejected": -0.9623022079467773, "logps/chosen": -400.1407470703125, "logps/rejected": -662.8094482421875, "loss": 0.557, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.7614247798919678, "rewards/margins": 1.2712466716766357, "rewards/rejected": -3.0326714515686035, "step": 3200 }, { "epoch": 0.47254526718680995, "grad_norm": 61.67686186772589, "learning_rate": 3.1682231855818565e-07, "logits/chosen": -0.9251901507377625, "logits/rejected": -0.8875066041946411, "logps/chosen": -382.7342224121094, "logps/rejected": -549.8390502929688, "loss": 0.4504, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.8720834255218506, "rewards/margins": 1.461350440979004, "rewards/rejected": -3.3334336280822754, "step": 3210 }, { "epoch": 0.47401737082290596, "grad_norm": 97.98450699752662, "learning_rate": 3.1558338742704175e-07, "logits/chosen": -1.0002073049545288, "logits/rejected": -0.7228268384933472, "logps/chosen": -517.6812133789062, "logps/rejected": -538.5872802734375, "loss": 0.6334, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -2.0547587871551514, "rewards/margins": 0.9009310007095337, "rewards/rejected": -2.9556899070739746, "step": 3220 }, { "epoch": 0.4754894744590019, "grad_norm": 92.16897393499394, "learning_rate": 3.1434272415250164e-07, "logits/chosen": -1.1355243921279907, "logits/rejected": -0.7260192632675171, "logps/chosen": -496.72674560546875, "logps/rejected": -601.2373046875, "loss": 0.5128, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.9694559574127197, "rewards/margins": 1.2598377466201782, "rewards/rejected": -3.2292933464050293, "step": 3230 }, { "epoch": 0.47696157809509787, "grad_norm": 46.80895832139151, "learning_rate": 3.1310036150211155e-07, "logits/chosen": -1.0410677194595337, "logits/rejected": -0.8257973790168762, "logps/chosen": -439.0824279785156, "logps/rejected": -550.9937744140625, "loss": 0.4721, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.5329301357269287, "rewards/margins": 1.3073517084121704, "rewards/rejected": -2.8402817249298096, "step": 3240 }, { "epoch": 0.4784336817311939, "grad_norm": 87.95062826503703, "learning_rate": 3.1185633228830076e-07, "logits/chosen": -1.271874189376831, "logits/rejected": -0.41823306679725647, "logps/chosen": -457.9461364746094, "logps/rejected": -551.9237060546875, "loss": 0.4721, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.5670020580291748, "rewards/margins": 1.43280827999115, "rewards/rejected": -2.9998106956481934, "step": 3250 }, { "epoch": 0.47990578536728984, "grad_norm": 66.92881169429357, "learning_rate": 3.1061066936751483e-07, "logits/chosen": -0.5986589193344116, "logits/rejected": -0.36469608545303345, "logps/chosen": -452.1084899902344, "logps/rejected": -606.2619018554688, "loss": 0.4009, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -1.9342241287231445, "rewards/margins": 1.2434868812561035, "rewards/rejected": -3.177711009979248, "step": 3260 }, { "epoch": 0.48137788900338585, "grad_norm": 103.75134336547266, "learning_rate": 3.0936340563934733e-07, "logits/chosen": -0.35131606459617615, "logits/rejected": 0.34717637300491333, "logps/chosen": -446.91717529296875, "logps/rejected": -612.5995483398438, "loss": 0.5381, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.9126863479614258, "rewards/margins": 1.7402470111846924, "rewards/rejected": -3.6529335975646973, "step": 3270 }, { "epoch": 0.4828499926394818, "grad_norm": 77.8948154573654, "learning_rate": 3.0811457404567153e-07, "logits/chosen": -0.7472583055496216, "logits/rejected": 0.11789913475513458, "logps/chosen": -519.041015625, "logps/rejected": -527.4039916992188, "loss": 0.4594, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.0146710872650146, "rewards/margins": 0.8171228170394897, "rewards/rejected": -2.831794023513794, "step": 3280 }, { "epoch": 0.4843220962755778, "grad_norm": 167.9212084252121, "learning_rate": 3.068642075697699e-07, "logits/chosen": -0.42541250586509705, "logits/rejected": 0.21263058483600616, "logps/chosen": -503.61920166015625, "logps/rejected": -599.8175048828125, "loss": 0.6, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -2.501636505126953, "rewards/margins": 1.1055630445480347, "rewards/rejected": -3.6071999073028564, "step": 3290 }, { "epoch": 0.4857941999116738, "grad_norm": 168.07406222164585, "learning_rate": 3.0561233923546336e-07, "logits/chosen": -0.4212028980255127, "logits/rejected": 0.12252716720104218, "logps/chosen": -459.73614501953125, "logps/rejected": -608.9522705078125, "loss": 0.51, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.2331066131591797, "rewards/margins": 1.4988734722137451, "rewards/rejected": -3.731980085372925, "step": 3300 }, { "epoch": 0.4872663035477698, "grad_norm": 72.64696076498669, "learning_rate": 3.0435900210623867e-07, "logits/chosen": -0.9463162422180176, "logits/rejected": -0.13293954730033875, "logps/chosen": -473.06561279296875, "logps/rejected": -623.835205078125, "loss": 0.4109, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -2.18930983543396, "rewards/margins": 1.4071576595306396, "rewards/rejected": -3.596467971801758, "step": 3310 }, { "epoch": 0.48873840718386574, "grad_norm": 76.13872340767301, "learning_rate": 3.031042292843755e-07, "logits/chosen": -0.7620629668235779, "logits/rejected": -0.11279682070016861, "logps/chosen": -505.63043212890625, "logps/rejected": -617.1547241210938, "loss": 0.5179, "rewards/accuracies": 0.9000000953674316, "rewards/chosen": -2.258166790008545, "rewards/margins": 1.1824322938919067, "rewards/rejected": -3.440598964691162, "step": 3320 }, { "epoch": 0.4902105108199617, "grad_norm": 124.287656631569, "learning_rate": 3.0184805391007205e-07, "logits/chosen": -0.7151976227760315, "logits/rejected": -0.8751751184463501, "logps/chosen": -550.53759765625, "logps/rejected": -651.2049560546875, "loss": 0.441, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -2.647606611251831, "rewards/margins": 0.7530060410499573, "rewards/rejected": -3.4006130695343018, "step": 3330 }, { "epoch": 0.4916826144560577, "grad_norm": 135.07782979997054, "learning_rate": 3.0059050916056977e-07, "logits/chosen": -1.12015700340271, "logits/rejected": -0.4860397279262543, "logps/chosen": -518.6348876953125, "logps/rejected": -528.7443237304688, "loss": 0.5541, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.6023058891296387, "rewards/margins": 0.8805994987487793, "rewards/rejected": -3.4829049110412598, "step": 3340 }, { "epoch": 0.49315471809215367, "grad_norm": 227.43220372603295, "learning_rate": 2.9933162824927713e-07, "logits/chosen": -1.0092880725860596, "logits/rejected": -0.72817462682724, "logps/chosen": -446.1585998535156, "logps/rejected": -555.7626953125, "loss": 0.581, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -2.2049596309661865, "rewards/margins": 1.1553776264190674, "rewards/rejected": -3.360337495803833, "step": 3350 }, { "epoch": 0.4946268217282497, "grad_norm": 113.59146684583767, "learning_rate": 2.9807144442489234e-07, "logits/chosen": -1.2836788892745972, "logits/rejected": -1.0262157917022705, "logps/chosen": -513.675537109375, "logps/rejected": -585.2378540039062, "loss": 0.4447, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -2.151932716369629, "rewards/margins": 0.9616401791572571, "rewards/rejected": -3.113572835922241, "step": 3360 }, { "epoch": 0.49609892536434563, "grad_norm": 151.2761705515275, "learning_rate": 2.9680999097052536e-07, "logits/chosen": -0.9294954538345337, "logits/rejected": -0.5569708943367004, "logps/chosen": -420.66619873046875, "logps/rejected": -587.71435546875, "loss": 0.479, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -2.234401226043701, "rewards/margins": 1.5371743440628052, "rewards/rejected": -3.771575450897217, "step": 3370 }, { "epoch": 0.49757102900044164, "grad_norm": 163.77758518356075, "learning_rate": 2.9554730120281847e-07, "logits/chosen": -0.9432289004325867, "logits/rejected": -0.5820508599281311, "logps/chosen": -520.8422241210938, "logps/rejected": -579.085693359375, "loss": 0.4814, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.5879578590393066, "rewards/margins": 1.0231280326843262, "rewards/rejected": -3.611085891723633, "step": 3380 }, { "epoch": 0.4990431326365376, "grad_norm": 118.552399882384, "learning_rate": 2.942834084710668e-07, "logits/chosen": -0.892548680305481, "logits/rejected": -0.5368214845657349, "logps/chosen": -583.15478515625, "logps/rejected": -600.8814697265625, "loss": 0.5559, "rewards/accuracies": 0.5666666626930237, "rewards/chosen": -3.060138702392578, "rewards/margins": 0.7581546306610107, "rewards/rejected": -3.8182930946350098, "step": 3390 }, { "epoch": 0.5005152362726336, "grad_norm": 61.18321683249202, "learning_rate": 2.9301834615633766e-07, "logits/chosen": -0.7124353051185608, "logits/rejected": -0.7218102216720581, "logps/chosen": -447.5498046875, "logps/rejected": -578.6387939453125, "loss": 0.5324, "rewards/accuracies": 0.7999998927116394, "rewards/chosen": -2.3925974369049072, "rewards/margins": 1.289864182472229, "rewards/rejected": -3.682461977005005, "step": 3400 }, { "epoch": 0.5019873399087296, "grad_norm": 70.14826603763085, "learning_rate": 2.917521476705879e-07, "logits/chosen": -1.1851778030395508, "logits/rejected": -0.8059045672416687, "logps/chosen": -569.7425537109375, "logps/rejected": -691.015869140625, "loss": 0.4258, "rewards/accuracies": 0.9000000953674316, "rewards/chosen": -2.613755702972412, "rewards/margins": 1.6566842794418335, "rewards/rejected": -4.270440101623535, "step": 3410 }, { "epoch": 0.5034594435448255, "grad_norm": 45.652088146296705, "learning_rate": 2.904848464557827e-07, "logits/chosen": -0.9869047403335571, "logits/rejected": -0.5741862654685974, "logps/chosen": -577.0040283203125, "logps/rejected": -618.1246948242188, "loss": 0.5622, "rewards/accuracies": 0.5, "rewards/chosen": -2.937192916870117, "rewards/margins": 0.7588188052177429, "rewards/rejected": -3.6960113048553467, "step": 3420 }, { "epoch": 0.5049315471809216, "grad_norm": 93.7953203447838, "learning_rate": 2.892164759830114e-07, "logits/chosen": -0.8094015121459961, "logits/rejected": -0.47796908020973206, "logps/chosen": -516.6316528320312, "logps/rejected": -595.246826171875, "loss": 0.4979, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.5407001972198486, "rewards/margins": 0.8587757349014282, "rewards/rejected": -3.3994762897491455, "step": 3430 }, { "epoch": 0.5064036508170175, "grad_norm": 86.41127829758365, "learning_rate": 2.8794706975160425e-07, "logits/chosen": -0.7042872309684753, "logits/rejected": 0.20396976172924042, "logps/chosen": -495.3429260253906, "logps/rejected": -618.5006713867188, "loss": 0.5474, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -2.5355000495910645, "rewards/margins": 1.2149041891098022, "rewards/rejected": -3.750403642654419, "step": 3440 }, { "epoch": 0.5078757544531135, "grad_norm": 164.93603292789112, "learning_rate": 2.866766612882468e-07, "logits/chosen": -1.3000671863555908, "logits/rejected": -0.9017229080200195, "logps/chosen": -479.858154296875, "logps/rejected": -591.9417724609375, "loss": 0.4891, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.1474249362945557, "rewards/margins": 0.9466966390609741, "rewards/rejected": -3.094121217727661, "step": 3450 }, { "epoch": 0.5093478580892095, "grad_norm": 96.37561146231899, "learning_rate": 2.8540528414609514e-07, "logits/chosen": -1.018848180770874, "logits/rejected": -0.7929521799087524, "logps/chosen": -447.865234375, "logps/rejected": -651.18701171875, "loss": 0.4804, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.2059645652770996, "rewards/margins": 1.5832328796386719, "rewards/rejected": -3.7891972064971924, "step": 3460 }, { "epoch": 0.5108199617253054, "grad_norm": 196.46591386606642, "learning_rate": 2.8413297190388947e-07, "logits/chosen": -1.0635197162628174, "logits/rejected": -0.8645036816596985, "logps/chosen": -504.1944885253906, "logps/rejected": -570.5606689453125, "loss": 0.4855, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -2.526508092880249, "rewards/margins": 0.9527558088302612, "rewards/rejected": -3.4792640209198, "step": 3470 }, { "epoch": 0.5122920653614015, "grad_norm": 133.05840098255638, "learning_rate": 2.8285975816506687e-07, "logits/chosen": -1.500643253326416, "logits/rejected": -1.4915549755096436, "logps/chosen": -579.1746215820312, "logps/rejected": -564.9222412109375, "loss": 0.4829, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.0425267219543457, "rewards/margins": 0.6850039958953857, "rewards/rejected": -2.7275304794311523, "step": 3480 }, { "epoch": 0.5137641689974974, "grad_norm": 114.26230847706957, "learning_rate": 2.815856765568746e-07, "logits/chosen": -1.5373557806015015, "logits/rejected": -1.3047198057174683, "logps/chosen": -454.78509521484375, "logps/rejected": -468.45477294921875, "loss": 0.4396, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.6142804622650146, "rewards/margins": 0.8521246910095215, "rewards/rejected": -2.466405153274536, "step": 3490 }, { "epoch": 0.5152362726335934, "grad_norm": 149.07762686642013, "learning_rate": 2.803107607294811e-07, "logits/chosen": -1.3443024158477783, "logits/rejected": -0.9240787625312805, "logps/chosen": -468.52276611328125, "logps/rejected": -563.1370239257812, "loss": 0.6109, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.1036009788513184, "rewards/margins": 1.0239379405975342, "rewards/rejected": -3.1275384426116943, "step": 3500 }, { "epoch": 0.5167083762696894, "grad_norm": 60.07938664565102, "learning_rate": 2.7903504435508787e-07, "logits/chosen": -1.2649275064468384, "logits/rejected": -1.0444282293319702, "logps/chosen": -524.1019287109375, "logps/rejected": -608.5594482421875, "loss": 0.6044, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.1403751373291016, "rewards/margins": 1.0087820291519165, "rewards/rejected": -3.1491570472717285, "step": 3510 }, { "epoch": 0.5181804799057854, "grad_norm": 82.85979150873672, "learning_rate": 2.777585611270398e-07, "logits/chosen": -1.093212604522705, "logits/rejected": -0.8458244204521179, "logps/chosen": -476.87091064453125, "logps/rejected": -545.1942138671875, "loss": 0.5018, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.1044373512268066, "rewards/margins": 1.140057921409607, "rewards/rejected": -3.244494676589966, "step": 3520 }, { "epoch": 0.5196525835418814, "grad_norm": 132.8237267241224, "learning_rate": 2.7648134475893544e-07, "logits/chosen": -1.1214923858642578, "logits/rejected": -0.8191500902175903, "logps/chosen": -443.44964599609375, "logps/rejected": -497.73419189453125, "loss": 0.5932, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.0088188648223877, "rewards/margins": 0.6817129850387573, "rewards/rejected": -2.6905322074890137, "step": 3530 }, { "epoch": 0.5211246871779773, "grad_norm": 101.9469424248708, "learning_rate": 2.7520342898373657e-07, "logits/chosen": -0.797698974609375, "logits/rejected": -0.7146264314651489, "logps/chosen": -411.5892639160156, "logps/rejected": -486.6763610839844, "loss": 0.5879, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -2.038975954055786, "rewards/margins": 0.7395116090774536, "rewards/rejected": -2.7784876823425293, "step": 3540 }, { "epoch": 0.5225967908140733, "grad_norm": 135.8145510125049, "learning_rate": 2.73924847552877e-07, "logits/chosen": -1.0406383275985718, "logits/rejected": -0.761631965637207, "logps/chosen": -420.68206787109375, "logps/rejected": -464.0370178222656, "loss": 0.5155, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.6280235052108765, "rewards/margins": 0.8213141560554504, "rewards/rejected": -2.449337959289551, "step": 3550 }, { "epoch": 0.5240688944501692, "grad_norm": 40.68831168502621, "learning_rate": 2.726456342353718e-07, "logits/chosen": -1.4041072130203247, "logits/rejected": -1.2757805585861206, "logps/chosen": -485.3564453125, "logps/rejected": -557.1806030273438, "loss": 0.4311, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.7481788396835327, "rewards/margins": 0.8918279409408569, "rewards/rejected": -2.6400065422058105, "step": 3560 }, { "epoch": 0.5255409980862653, "grad_norm": 86.82123480554844, "learning_rate": 2.7136582281692443e-07, "logits/chosen": -0.9888653755187988, "logits/rejected": -0.7883027195930481, "logps/chosen": -455.83990478515625, "logps/rejected": -528.5462646484375, "loss": 0.4741, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -1.8590036630630493, "rewards/margins": 1.416704773902893, "rewards/rejected": -3.2757084369659424, "step": 3570 }, { "epoch": 0.5270131017223613, "grad_norm": 66.2519746442281, "learning_rate": 2.700854470990357e-07, "logits/chosen": -0.8329097032546997, "logits/rejected": -0.5545793175697327, "logps/chosen": -493.0042419433594, "logps/rejected": -610.2681274414062, "loss": 0.548, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -2.285214900970459, "rewards/margins": 1.2870739698410034, "rewards/rejected": -3.572288990020752, "step": 3580 }, { "epoch": 0.5284852053584572, "grad_norm": 94.05341844759623, "learning_rate": 2.6880454089810954e-07, "logits/chosen": -1.2370531558990479, "logits/rejected": -0.6671432256698608, "logps/chosen": -491.1453552246094, "logps/rejected": -555.08203125, "loss": 0.5699, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.9788224697113037, "rewards/margins": 0.8359730839729309, "rewards/rejected": -2.8147952556610107, "step": 3590 }, { "epoch": 0.5299573089945532, "grad_norm": 106.15463681434012, "learning_rate": 2.6752313804456124e-07, "logits/chosen": -1.1055041551589966, "logits/rejected": -0.6545578241348267, "logps/chosen": -398.18572998046875, "logps/rejected": -457.91021728515625, "loss": 0.4474, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -1.7753149271011353, "rewards/margins": 1.1495778560638428, "rewards/rejected": -2.9248929023742676, "step": 3600 }, { "epoch": 0.5314294126306492, "grad_norm": 51.42329147526349, "learning_rate": 2.66241272381923e-07, "logits/chosen": -1.4011653661727905, "logits/rejected": -0.9820989370346069, "logps/chosen": -487.7442321777344, "logps/rejected": -488.4327697753906, "loss": 0.475, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -2.4035143852233887, "rewards/margins": 0.5535462498664856, "rewards/rejected": -2.9570605754852295, "step": 3610 }, { "epoch": 0.5329015162667452, "grad_norm": 83.07618658077477, "learning_rate": 2.649589777659506e-07, "logits/chosen": -1.0022094249725342, "logits/rejected": -0.9162474870681763, "logps/chosen": -403.79388427734375, "logps/rejected": -632.6209716796875, "loss": 0.4923, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -2.006913423538208, "rewards/margins": 1.435204267501831, "rewards/rejected": -3.442117691040039, "step": 3620 }, { "epoch": 0.5343736199028412, "grad_norm": 95.48613411816375, "learning_rate": 2.6367628806372893e-07, "logits/chosen": -1.3434334993362427, "logits/rejected": -0.6106802821159363, "logps/chosen": -589.4462280273438, "logps/rejected": -620.58349609375, "loss": 0.5333, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -2.4060192108154297, "rewards/margins": 1.289965271949768, "rewards/rejected": -3.695984363555908, "step": 3630 }, { "epoch": 0.5358457235389371, "grad_norm": 88.94966670076559, "learning_rate": 2.623932371527776e-07, "logits/chosen": -1.4550727605819702, "logits/rejected": -0.5974916219711304, "logps/chosen": -512.2884521484375, "logps/rejected": -567.3715209960938, "loss": 0.5042, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.9926986694335938, "rewards/margins": 1.441896915435791, "rewards/rejected": -3.434595823287964, "step": 3640 }, { "epoch": 0.5373178271750331, "grad_norm": 144.70516586885373, "learning_rate": 2.611098589201563e-07, "logits/chosen": -0.8839343786239624, "logits/rejected": -0.6502280831336975, "logps/chosen": -454.4888610839844, "logps/rejected": -609.1740112304688, "loss": 0.4138, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -2.3977155685424805, "rewards/margins": 1.6403570175170898, "rewards/rejected": -4.0380730628967285, "step": 3650 }, { "epoch": 0.5387899308111291, "grad_norm": 322.08883595138076, "learning_rate": 2.5982618726156965e-07, "logits/chosen": -1.1795523166656494, "logits/rejected": -0.8610784411430359, "logps/chosen": -436.20977783203125, "logps/rejected": -571.6282958984375, "loss": 0.4293, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -2.2965991497039795, "rewards/margins": 1.3638889789581299, "rewards/rejected": -3.6604881286621094, "step": 3660 }, { "epoch": 0.5402620344472251, "grad_norm": 79.91620253175974, "learning_rate": 2.5854225608047217e-07, "logits/chosen": -1.0248043537139893, "logits/rejected": -0.8329952955245972, "logps/chosen": -502.764892578125, "logps/rejected": -602.9068603515625, "loss": 0.4291, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.441542625427246, "rewards/margins": 1.0967991352081299, "rewards/rejected": -3.538341522216797, "step": 3670 }, { "epoch": 0.541734138083321, "grad_norm": 141.46441662539445, "learning_rate": 2.572580992871725e-07, "logits/chosen": -1.5260649919509888, "logits/rejected": -1.1740323305130005, "logps/chosen": -588.5630493164062, "logps/rejected": -625.4890747070312, "loss": 0.5023, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.2363507747650146, "rewards/margins": 0.9393233060836792, "rewards/rejected": -3.1756739616394043, "step": 3680 }, { "epoch": 0.543206241719417, "grad_norm": 104.28666697073605, "learning_rate": 2.5597375079793826e-07, "logits/chosen": -1.2928651571273804, "logits/rejected": -0.8324087262153625, "logps/chosen": -412.5162048339844, "logps/rejected": -518.4513549804688, "loss": 0.5089, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.765946626663208, "rewards/margins": 0.946916401386261, "rewards/rejected": -2.712862730026245, "step": 3690 }, { "epoch": 0.5446783453555131, "grad_norm": 130.83779984210676, "learning_rate": 2.5468924453409977e-07, "logits/chosen": -1.262528896331787, "logits/rejected": -0.9617989659309387, "logps/chosen": -470.62945556640625, "logps/rejected": -536.2945556640625, "loss": 0.7204, "rewards/accuracies": 0.5333333015441895, "rewards/chosen": -2.2758002281188965, "rewards/margins": 0.6281536817550659, "rewards/rejected": -2.903953790664673, "step": 3700 }, { "epoch": 0.546150448991609, "grad_norm": 152.95755861823764, "learning_rate": 2.534046144211544e-07, "logits/chosen": -1.5449644327163696, "logits/rejected": -1.4245322942733765, "logps/chosen": -471.7792053222656, "logps/rejected": -541.1217651367188, "loss": 0.5859, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.8622760772705078, "rewards/margins": 0.9120270609855652, "rewards/rejected": -2.7743031978607178, "step": 3710 }, { "epoch": 0.547622552627705, "grad_norm": 59.17155786640775, "learning_rate": 2.521198943878708e-07, "logits/chosen": -1.4385583400726318, "logits/rejected": -1.7578184604644775, "logps/chosen": -440.26641845703125, "logps/rejected": -546.3853759765625, "loss": 0.5733, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -1.8807086944580078, "rewards/margins": 0.3880549967288971, "rewards/rejected": -2.268763780593872, "step": 3720 }, { "epoch": 0.5490946562638009, "grad_norm": 61.881472655644686, "learning_rate": 2.5083511836539213e-07, "logits/chosen": -1.77863347530365, "logits/rejected": -1.6871923208236694, "logps/chosen": -429.9437561035156, "logps/rejected": -447.72186279296875, "loss": 0.4463, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.5609115362167358, "rewards/margins": 0.4274429380893707, "rewards/rejected": -1.9883544445037842, "step": 3730 }, { "epoch": 0.550566759899897, "grad_norm": 90.66992302484171, "learning_rate": 2.495503202863407e-07, "logits/chosen": -1.6335229873657227, "logits/rejected": -1.4598299264907837, "logps/chosen": -396.29583740234375, "logps/rejected": -454.7091369628906, "loss": 0.5254, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.6245256662368774, "rewards/margins": 0.949206531047821, "rewards/rejected": -2.5737321376800537, "step": 3740 }, { "epoch": 0.552038863535993, "grad_norm": 45.57917448479132, "learning_rate": 2.4826553408392104e-07, "logits/chosen": -1.6143033504486084, "logits/rejected": -1.2706949710845947, "logps/chosen": -380.66241455078125, "logps/rejected": -499.0823669433594, "loss": 0.4934, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.3969308137893677, "rewards/margins": 1.380838394165039, "rewards/rejected": -2.7777693271636963, "step": 3750 }, { "epoch": 0.5535109671720889, "grad_norm": 106.56612027161793, "learning_rate": 2.469807936910242e-07, "logits/chosen": -1.5957189798355103, "logits/rejected": -1.3597112894058228, "logps/chosen": -441.5704040527344, "logps/rejected": -518.8150634765625, "loss": 0.4738, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.4410152435302734, "rewards/margins": 1.0514957904815674, "rewards/rejected": -2.49251127243042, "step": 3760 }, { "epoch": 0.5549830708081849, "grad_norm": 82.56240515825458, "learning_rate": 2.456961330393313e-07, "logits/chosen": -1.5211899280548096, "logits/rejected": -1.0843480825424194, "logps/chosen": -438.5062561035156, "logps/rejected": -453.1576232910156, "loss": 0.5257, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -1.4770877361297607, "rewards/margins": 0.8786066174507141, "rewards/rejected": -2.35569429397583, "step": 3770 }, { "epoch": 0.5564551744442808, "grad_norm": 65.3512682214412, "learning_rate": 2.444115860584174e-07, "logits/chosen": -1.2876940965652466, "logits/rejected": -0.9141214489936829, "logps/chosen": -478.1373596191406, "logps/rejected": -567.1768798828125, "loss": 0.4454, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.8468248844146729, "rewards/margins": 1.1089718341827393, "rewards/rejected": -2.955796480178833, "step": 3780 }, { "epoch": 0.5579272780803769, "grad_norm": 99.23231842653799, "learning_rate": 2.4312718667485523e-07, "logits/chosen": -1.4020931720733643, "logits/rejected": -1.1753932237625122, "logps/chosen": -404.2585144042969, "logps/rejected": -488.93670654296875, "loss": 0.4067, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -1.532779574394226, "rewards/margins": 1.1545389890670776, "rewards/rejected": -2.6873185634613037, "step": 3790 }, { "epoch": 0.5593993817164729, "grad_norm": 100.61534189513759, "learning_rate": 2.418429688113194e-07, "logits/chosen": -1.2818669080734253, "logits/rejected": -0.9343087077140808, "logps/chosen": -598.4937744140625, "logps/rejected": -568.2569580078125, "loss": 0.4878, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.617891788482666, "rewards/margins": 0.8785541653633118, "rewards/rejected": -3.496445894241333, "step": 3800 }, { "epoch": 0.5608714853525688, "grad_norm": 125.06370873034457, "learning_rate": 2.405589663856904e-07, "logits/chosen": -1.0534967184066772, "logits/rejected": -0.796125054359436, "logps/chosen": -456.1944274902344, "logps/rejected": -579.9053955078125, "loss": 0.462, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.9744675159454346, "rewards/margins": 1.4265060424804688, "rewards/rejected": -3.4009735584259033, "step": 3810 }, { "epoch": 0.5623435889886648, "grad_norm": 59.73602878608555, "learning_rate": 2.3927521331015865e-07, "logits/chosen": -1.0593222379684448, "logits/rejected": -0.7076674699783325, "logps/chosen": -380.1196594238281, "logps/rejected": -539.4362182617188, "loss": 0.4875, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.722365140914917, "rewards/margins": 1.5474141836166382, "rewards/rejected": -3.2697787284851074, "step": 3820 }, { "epoch": 0.5638156926247608, "grad_norm": 79.65260420030506, "learning_rate": 2.379917434903289e-07, "logits/chosen": -0.9202688336372375, "logits/rejected": -0.8789048194885254, "logps/chosen": -527.4368286132812, "logps/rejected": -653.8619384765625, "loss": 0.4841, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.5801780223846436, "rewards/margins": 1.1735316514968872, "rewards/rejected": -3.753709316253662, "step": 3830 }, { "epoch": 0.5652877962608568, "grad_norm": 105.94838214690745, "learning_rate": 2.3670859082432458e-07, "logits/chosen": -1.366370439529419, "logits/rejected": -1.1931499242782593, "logps/chosen": -533.2824096679688, "logps/rejected": -643.5277099609375, "loss": 0.523, "rewards/accuracies": 0.9000000953674316, "rewards/chosen": -2.2342772483825684, "rewards/margins": 1.7616726160049438, "rewards/rejected": -3.995950222015381, "step": 3840 }, { "epoch": 0.5667598998969527, "grad_norm": 66.40181477677648, "learning_rate": 2.35425789201893e-07, "logits/chosen": -0.9468305706977844, "logits/rejected": -0.4881543219089508, "logps/chosen": -535.8870239257812, "logps/rejected": -688.7952270507812, "loss": 0.416, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.5945944786071777, "rewards/margins": 1.6216539144515991, "rewards/rejected": -4.216248512268066, "step": 3850 }, { "epoch": 0.5682320035330487, "grad_norm": 57.94401991168832, "learning_rate": 2.3414337250350982e-07, "logits/chosen": -0.9698936343193054, "logits/rejected": -0.698264479637146, "logps/chosen": -499.48114013671875, "logps/rejected": -620.9559326171875, "loss": 0.5635, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -2.667584180831909, "rewards/margins": 1.292751669883728, "rewards/rejected": -3.960336208343506, "step": 3860 }, { "epoch": 0.5697041071691447, "grad_norm": 120.33478274769186, "learning_rate": 2.3286137459948428e-07, "logits/chosen": -1.1415274143218994, "logits/rejected": -0.9294644594192505, "logps/chosen": -384.8303527832031, "logps/rejected": -570.9527587890625, "loss": 0.5407, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.9156959056854248, "rewards/margins": 1.6666879653930664, "rewards/rejected": -3.582383632659912, "step": 3870 }, { "epoch": 0.5711762108052407, "grad_norm": 113.35472760156007, "learning_rate": 2.3157982934906463e-07, "logits/chosen": -1.6657747030258179, "logits/rejected": -1.5370306968688965, "logps/chosen": -462.897705078125, "logps/rejected": -558.5958862304688, "loss": 0.4491, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.4978281259536743, "rewards/margins": 1.1867599487304688, "rewards/rejected": -2.6845879554748535, "step": 3880 }, { "epoch": 0.5726483144413367, "grad_norm": 57.025811267457534, "learning_rate": 2.3029877059954414e-07, "logits/chosen": -1.7329070568084717, "logits/rejected": -1.3369855880737305, "logps/chosen": -398.78497314453125, "logps/rejected": -550.7677001953125, "loss": 0.4763, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.4718331098556519, "rewards/margins": 1.6102197170257568, "rewards/rejected": -3.0820529460906982, "step": 3890 }, { "epoch": 0.5741204180774326, "grad_norm": 106.08643711485834, "learning_rate": 2.2901823218536693e-07, "logits/chosen": -1.489159345626831, "logits/rejected": -1.0795636177062988, "logps/chosen": -426.4410705566406, "logps/rejected": -435.78369140625, "loss": 0.5212, "rewards/accuracies": 0.5666666626930237, "rewards/chosen": -1.8088414669036865, "rewards/margins": 0.6170053482055664, "rewards/rejected": -2.425846815109253, "step": 3900 }, { "epoch": 0.5755925217135286, "grad_norm": 54.81465542460915, "learning_rate": 2.2773824792723428e-07, "logits/chosen": -1.1983683109283447, "logits/rejected": -1.030940055847168, "logps/chosen": -374.3813171386719, "logps/rejected": -536.5989379882812, "loss": 0.4032, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -1.4547827243804932, "rewards/margins": 1.6514374017715454, "rewards/rejected": -3.106220245361328, "step": 3910 }, { "epoch": 0.5770646253496247, "grad_norm": 131.8046182513765, "learning_rate": 2.2645885163121156e-07, "logits/chosen": -1.4867804050445557, "logits/rejected": -1.2189862728118896, "logps/chosen": -443.75689697265625, "logps/rejected": -578.314208984375, "loss": 0.4408, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -1.6495990753173828, "rewards/margins": 1.5253831148147583, "rewards/rejected": -3.1749820709228516, "step": 3920 }, { "epoch": 0.5785367289857206, "grad_norm": 88.47104333244829, "learning_rate": 2.2518007708783512e-07, "logits/chosen": -1.2885537147521973, "logits/rejected": -0.8970157504081726, "logps/chosen": -467.15289306640625, "logps/rejected": -679.8993530273438, "loss": 0.5568, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.0828945636749268, "rewards/margins": 1.580463171005249, "rewards/rejected": -3.6633574962615967, "step": 3930 }, { "epoch": 0.5800088326218166, "grad_norm": 144.53884960188145, "learning_rate": 2.2390195807122027e-07, "logits/chosen": -1.1762340068817139, "logits/rejected": -1.1747605800628662, "logps/chosen": -401.1009826660156, "logps/rejected": -435.6044006347656, "loss": 0.5344, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -1.4244767427444458, "rewards/margins": 0.6796760559082031, "rewards/rejected": -2.1041529178619385, "step": 3940 }, { "epoch": 0.5814809362579125, "grad_norm": 97.47022453059363, "learning_rate": 2.2262452833816886e-07, "logits/chosen": -1.1835429668426514, "logits/rejected": -0.4401678442955017, "logps/chosen": -419.27301025390625, "logps/rejected": -492.27447509765625, "loss": 0.551, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.656942367553711, "rewards/margins": 1.1906611919403076, "rewards/rejected": -2.8476035594940186, "step": 3950 }, { "epoch": 0.5829530398940085, "grad_norm": 76.22444315682189, "learning_rate": 2.2134782162727778e-07, "logits/chosen": -1.3269715309143066, "logits/rejected": -1.03289794921875, "logps/chosen": -398.5579528808594, "logps/rejected": -494.4390563964844, "loss": 0.546, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.5403027534484863, "rewards/margins": 1.2021934986114502, "rewards/rejected": -2.7424960136413574, "step": 3960 }, { "epoch": 0.5844251435301046, "grad_norm": 91.33028683132093, "learning_rate": 2.2007187165804822e-07, "logits/chosen": -1.200577974319458, "logits/rejected": -0.8154312968254089, "logps/chosen": -441.831298828125, "logps/rejected": -612.3999633789062, "loss": 0.576, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.56230628490448, "rewards/margins": 1.3954347372055054, "rewards/rejected": -2.9577410221099854, "step": 3970 }, { "epoch": 0.5858972471662005, "grad_norm": 120.67388338591378, "learning_rate": 2.1879671212999437e-07, "logits/chosen": -1.3850030899047852, "logits/rejected": -1.1487380266189575, "logps/chosen": -394.9458923339844, "logps/rejected": -439.26837158203125, "loss": 0.4756, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -1.2616503238677979, "rewards/margins": 0.9496402740478516, "rewards/rejected": -2.2112908363342285, "step": 3980 }, { "epoch": 0.5873693508022965, "grad_norm": 64.13137302467265, "learning_rate": 2.1752237672175433e-07, "logits/chosen": -1.5979244709014893, "logits/rejected": -1.2981550693511963, "logps/chosen": -412.94976806640625, "logps/rejected": -476.72186279296875, "loss": 0.496, "rewards/accuracies": 0.6333333849906921, "rewards/chosen": -1.4475165605545044, "rewards/margins": 0.658544659614563, "rewards/rejected": -2.1060609817504883, "step": 3990 }, { "epoch": 0.5888414544383924, "grad_norm": 88.69610580552754, "learning_rate": 2.162488990901998e-07, "logits/chosen": -1.286516785621643, "logits/rejected": -1.230358362197876, "logps/chosen": -462.65252685546875, "logps/rejected": -565.0322265625, "loss": 0.4958, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.040295124053955, "rewards/margins": 1.1627228260040283, "rewards/rejected": -3.2030181884765625, "step": 4000 }, { "epoch": 0.5903135580744885, "grad_norm": 82.8018643796261, "learning_rate": 2.1497631286954764e-07, "logits/chosen": -1.5582008361816406, "logits/rejected": -1.0771392583847046, "logps/chosen": -487.6258850097656, "logps/rejected": -536.5396728515625, "loss": 0.4859, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.5401533842086792, "rewards/margins": 1.3004016876220703, "rewards/rejected": -2.840555191040039, "step": 4010 }, { "epoch": 0.5917856617105844, "grad_norm": 164.80643813291522, "learning_rate": 2.1370465167047118e-07, "logits/chosen": -1.5033612251281738, "logits/rejected": -1.1781336069107056, "logps/chosen": -415.71392822265625, "logps/rejected": -462.5985412597656, "loss": 0.49, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.7596426010131836, "rewards/margins": 0.992581844329834, "rewards/rejected": -2.7522244453430176, "step": 4020 }, { "epoch": 0.5932577653466804, "grad_norm": 59.8446287734588, "learning_rate": 2.124339490792128e-07, "logits/chosen": -1.4557219743728638, "logits/rejected": -1.3001980781555176, "logps/chosen": -452.73126220703125, "logps/rejected": -607.0549926757812, "loss": 0.4293, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.842187523841858, "rewards/margins": 1.4325052499771118, "rewards/rejected": -3.2746925354003906, "step": 4030 }, { "epoch": 0.5947298689827764, "grad_norm": 99.88325490179106, "learning_rate": 2.1116423865669703e-07, "logits/chosen": -1.559486746788025, "logits/rejected": -0.9255777597427368, "logps/chosen": -440.6299743652344, "logps/rejected": -545.955078125, "loss": 0.4867, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.7578004598617554, "rewards/margins": 1.2535126209259033, "rewards/rejected": -3.011312961578369, "step": 4040 }, { "epoch": 0.5962019726188724, "grad_norm": 89.91731817292883, "learning_rate": 2.0989555393764354e-07, "logits/chosen": -1.293605923652649, "logits/rejected": -1.3127516508102417, "logps/chosen": -565.013671875, "logps/rejected": -562.1390380859375, "loss": 0.5946, "rewards/accuracies": 0.5333333611488342, "rewards/chosen": -1.9389889240264893, "rewards/margins": 0.5274927616119385, "rewards/rejected": -2.4664814472198486, "step": 4050 }, { "epoch": 0.5976740762549684, "grad_norm": 75.61195520655684, "learning_rate": 2.0862792842968214e-07, "logits/chosen": -1.2534714937210083, "logits/rejected": -0.6822981834411621, "logps/chosen": -474.09326171875, "logps/rejected": -471.49560546875, "loss": 0.5203, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.6415497064590454, "rewards/margins": 0.8943950533866882, "rewards/rejected": -2.535945177078247, "step": 4060 }, { "epoch": 0.5991461798910643, "grad_norm": 74.48009054349463, "learning_rate": 2.0736139561246713e-07, "logits/chosen": -1.2605535984039307, "logits/rejected": -0.9497723579406738, "logps/chosen": -429.6415100097656, "logps/rejected": -536.9093627929688, "loss": 0.5403, "rewards/accuracies": 0.5999999642372131, "rewards/chosen": -1.7722374200820923, "rewards/margins": 0.9319728016853333, "rewards/rejected": -2.704209804534912, "step": 4070 }, { "epoch": 0.6006182835271603, "grad_norm": 167.6179453167691, "learning_rate": 2.060959889367938e-07, "logits/chosen": -1.4208152294158936, "logits/rejected": -1.0713694095611572, "logps/chosen": -444.0927734375, "logps/rejected": -618.0239868164062, "loss": 0.5208, "rewards/accuracies": 0.9333333969116211, "rewards/chosen": -1.594525694847107, "rewards/margins": 1.8230946063995361, "rewards/rejected": -3.4176204204559326, "step": 4080 }, { "epoch": 0.6020903871632562, "grad_norm": 58.42146368479048, "learning_rate": 2.0483174182371435e-07, "logits/chosen": -1.2485980987548828, "logits/rejected": -1.230147123336792, "logps/chosen": -447.17877197265625, "logps/rejected": -491.56512451171875, "loss": 0.5198, "rewards/accuracies": 0.4999999403953552, "rewards/chosen": -1.8198697566986084, "rewards/margins": 0.40783554315567017, "rewards/rejected": -2.227705478668213, "step": 4090 }, { "epoch": 0.6035624907993523, "grad_norm": 77.7073924106762, "learning_rate": 2.0356868766365536e-07, "logits/chosen": -1.4538729190826416, "logits/rejected": -1.2259035110473633, "logps/chosen": -473.74658203125, "logps/rejected": -567.4500732421875, "loss": 0.5613, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.8328685760498047, "rewards/margins": 0.5852871537208557, "rewards/rejected": -2.4181556701660156, "step": 4100 }, { "epoch": 0.6050345944354483, "grad_norm": 47.20760639602046, "learning_rate": 2.023068598155363e-07, "logits/chosen": -1.5401443243026733, "logits/rejected": -1.2264618873596191, "logps/chosen": -512.0546264648438, "logps/rejected": -492.6022033691406, "loss": 0.5921, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.111586809158325, "rewards/margins": 0.5268180966377258, "rewards/rejected": -2.638404369354248, "step": 4110 }, { "epoch": 0.6065066980715442, "grad_norm": 166.3540160109531, "learning_rate": 2.010462916058875e-07, "logits/chosen": -1.5501317977905273, "logits/rejected": -1.3588258028030396, "logps/chosen": -495.94903564453125, "logps/rejected": -630.6212158203125, "loss": 0.4753, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -1.701025366783142, "rewards/margins": 1.473243236541748, "rewards/rejected": -3.174268960952759, "step": 4120 }, { "epoch": 0.6079788017076402, "grad_norm": 76.88460088192504, "learning_rate": 1.9978701632797118e-07, "logits/chosen": -1.4404414892196655, "logits/rejected": -1.3681303262710571, "logps/chosen": -391.15899658203125, "logps/rejected": -478.22723388671875, "loss": 0.4961, "rewards/accuracies": 0.76666659116745, "rewards/chosen": -1.62236750125885, "rewards/margins": 0.7691271901130676, "rewards/rejected": -2.3914947509765625, "step": 4130 }, { "epoch": 0.6094509053437362, "grad_norm": 78.04768322916944, "learning_rate": 1.9852906724090127e-07, "logits/chosen": -1.5495954751968384, "logits/rejected": -1.3348801136016846, "logps/chosen": -378.16619873046875, "logps/rejected": -523.1204223632812, "loss": 0.4102, "rewards/accuracies": 0.73333340883255, "rewards/chosen": -1.6241337060928345, "rewards/margins": 1.2306989431381226, "rewards/rejected": -2.854832410812378, "step": 4140 }, { "epoch": 0.6109230089798322, "grad_norm": 71.77760045629627, "learning_rate": 1.9727247756876534e-07, "logits/chosen": -1.3857651948928833, "logits/rejected": -1.4891043901443481, "logps/chosen": -376.3533020019531, "logps/rejected": -423.8584899902344, "loss": 0.5789, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.4472265243530273, "rewards/margins": 0.23889613151550293, "rewards/rejected": -1.6861225366592407, "step": 4150 }, { "epoch": 0.6123951126159282, "grad_norm": 154.44237844465218, "learning_rate": 1.9601728049974683e-07, "logits/chosen": -1.0164905786514282, "logits/rejected": -0.9828926920890808, "logps/chosen": -404.97601318359375, "logps/rejected": -601.942626953125, "loss": 0.4779, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.8876426219940186, "rewards/margins": 1.514784574508667, "rewards/rejected": -3.4024271965026855, "step": 4160 }, { "epoch": 0.6138672162520241, "grad_norm": 141.1191225899058, "learning_rate": 1.94763509185249e-07, "logits/chosen": -1.2642747163772583, "logits/rejected": -0.9743534922599792, "logps/chosen": -449.161865234375, "logps/rejected": -586.4234619140625, "loss": 0.4459, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.940500020980835, "rewards/margins": 1.268189787864685, "rewards/rejected": -3.2086899280548096, "step": 4170 }, { "epoch": 0.6153393198881201, "grad_norm": 94.6393683561518, "learning_rate": 1.935111967390189e-07, "logits/chosen": -1.464521050453186, "logits/rejected": -0.9969412088394165, "logps/chosen": -416.349365234375, "logps/rejected": -506.1065368652344, "loss": 0.4172, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.7117770910263062, "rewards/margins": 1.4608806371688843, "rewards/rejected": -3.1726577281951904, "step": 4180 }, { "epoch": 0.6168114235242161, "grad_norm": 144.02589303977794, "learning_rate": 1.922603762362729e-07, "logits/chosen": -1.304671049118042, "logits/rejected": -0.6768732070922852, "logps/chosen": -444.5458984375, "logps/rejected": -567.3345947265625, "loss": 0.5234, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.1895623207092285, "rewards/margins": 1.1984633207321167, "rewards/rejected": -3.3880257606506348, "step": 4190 }, { "epoch": 0.6182835271603121, "grad_norm": 152.47598774250133, "learning_rate": 1.9101108071282342e-07, "logits/chosen": -1.3892663717269897, "logits/rejected": -0.9998384714126587, "logps/chosen": -502.8368225097656, "logps/rejected": -621.2618408203125, "loss": 0.4721, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -2.2931313514709473, "rewards/margins": 1.647463083267212, "rewards/rejected": -3.94059419631958, "step": 4200 }, { "epoch": 0.619755630796408, "grad_norm": 209.9216993920297, "learning_rate": 1.8976334316420576e-07, "logits/chosen": -1.6642833948135376, "logits/rejected": -1.2484536170959473, "logps/chosen": -546.4644165039062, "logps/rejected": -595.1619262695312, "loss": 0.4791, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.100569009780884, "rewards/margins": 1.0795539617538452, "rewards/rejected": -3.1801230907440186, "step": 4210 }, { "epoch": 0.621227734432504, "grad_norm": 94.85024625133835, "learning_rate": 1.8851719654480748e-07, "logits/chosen": -1.5108282566070557, "logits/rejected": -1.0989423990249634, "logps/chosen": -447.67327880859375, "logps/rejected": -557.6844482421875, "loss": 0.4872, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.9102060794830322, "rewards/margins": 1.181675910949707, "rewards/rejected": -3.0918819904327393, "step": 4220 }, { "epoch": 0.6226998380686001, "grad_norm": 144.09810984087537, "learning_rate": 1.8727267376699735e-07, "logits/chosen": -1.419729471206665, "logits/rejected": -0.9745621681213379, "logps/chosen": -510.80047607421875, "logps/rejected": -543.3255615234375, "loss": 0.5549, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.880112886428833, "rewards/margins": 1.0943782329559326, "rewards/rejected": -2.9744911193847656, "step": 4230 }, { "epoch": 0.624171941704696, "grad_norm": 65.32185352345809, "learning_rate": 1.8602980770025645e-07, "logits/chosen": -1.2146351337432861, "logits/rejected": -0.7923814058303833, "logps/chosen": -451.31658935546875, "logps/rejected": -515.2090454101562, "loss": 0.4314, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.5133302211761475, "rewards/margins": 1.4617403745651245, "rewards/rejected": -2.9750709533691406, "step": 4240 }, { "epoch": 0.625644045340792, "grad_norm": 400.9336865441555, "learning_rate": 1.8478863117031007e-07, "logits/chosen": -1.047730565071106, "logits/rejected": -0.43196743726730347, "logps/chosen": -457.43035888671875, "logps/rejected": -635.474609375, "loss": 0.5508, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -2.315809488296509, "rewards/margins": 1.5847208499908447, "rewards/rejected": -3.9005303382873535, "step": 4250 }, { "epoch": 0.6271161489768879, "grad_norm": 190.76353636910375, "learning_rate": 1.8354917695826026e-07, "logits/chosen": -1.206991195678711, "logits/rejected": -1.0310016870498657, "logps/chosen": -580.2699584960938, "logps/rejected": -672.878662109375, "loss": 0.4824, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.416865825653076, "rewards/margins": 1.251690149307251, "rewards/rejected": -3.668555736541748, "step": 4260 }, { "epoch": 0.6285882526129839, "grad_norm": 68.9692743255753, "learning_rate": 1.8231147779972074e-07, "logits/chosen": -0.866108238697052, "logits/rejected": -0.6675370931625366, "logps/chosen": -403.5827331542969, "logps/rejected": -582.376953125, "loss": 0.5206, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.99154531955719, "rewards/margins": 1.4701696634292603, "rewards/rejected": -3.46171498298645, "step": 4270 }, { "epoch": 0.63006035624908, "grad_norm": 44.600304662020214, "learning_rate": 1.8107556638395168e-07, "logits/chosen": -1.6005884408950806, "logits/rejected": -1.0544960498809814, "logps/chosen": -445.9622497558594, "logps/rejected": -620.0938720703125, "loss": 0.4594, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.6346498727798462, "rewards/margins": 1.7904284000396729, "rewards/rejected": -3.4250781536102295, "step": 4280 }, { "epoch": 0.6315324598851759, "grad_norm": 125.88691549754662, "learning_rate": 1.798414753529971e-07, "logits/chosen": -1.414941668510437, "logits/rejected": -1.3350245952606201, "logps/chosen": -474.35955810546875, "logps/rejected": -544.1416625976562, "loss": 0.4221, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.7634519338607788, "rewards/margins": 1.0667226314544678, "rewards/rejected": -2.830174684524536, "step": 4290 }, { "epoch": 0.6330045635212719, "grad_norm": 95.99487386342098, "learning_rate": 1.7860923730082152e-07, "logits/chosen": -1.4767735004425049, "logits/rejected": -1.374411940574646, "logps/chosen": -537.554443359375, "logps/rejected": -551.3592529296875, "loss": 0.5782, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.1924350261688232, "rewards/margins": 0.7634781002998352, "rewards/rejected": -2.9559130668640137, "step": 4300 }, { "epoch": 0.6344766671573678, "grad_norm": 62.96501205963991, "learning_rate": 1.7737888477245052e-07, "logits/chosen": -1.388330101966858, "logits/rejected": -1.2299821376800537, "logps/chosen": -537.3853149414062, "logps/rejected": -637.923095703125, "loss": 0.4149, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.465048313140869, "rewards/margins": 1.0545275211334229, "rewards/rejected": -3.519575834274292, "step": 4310 }, { "epoch": 0.6359487707934639, "grad_norm": 141.6486197599099, "learning_rate": 1.761504502631102e-07, "logits/chosen": -1.5358697175979614, "logits/rejected": -1.291987657546997, "logps/chosen": -504.5132751464844, "logps/rejected": -681.7574462890625, "loss": 0.5659, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -2.266136646270752, "rewards/margins": 1.7751989364624023, "rewards/rejected": -4.041335582733154, "step": 4320 }, { "epoch": 0.6374208744295599, "grad_norm": 175.4677207040497, "learning_rate": 1.749239662173693e-07, "logits/chosen": -1.0412142276763916, "logits/rejected": -0.8617256283760071, "logps/chosen": -406.6339111328125, "logps/rejected": -548.8243408203125, "loss": 0.4149, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.2163126468658447, "rewards/margins": 1.2508987188339233, "rewards/rejected": -3.4672112464904785, "step": 4330 }, { "epoch": 0.6388929780656558, "grad_norm": 136.91063087882577, "learning_rate": 1.7369946502828245e-07, "logits/chosen": -1.4333940744400024, "logits/rejected": -1.0822374820709229, "logps/chosen": -430.1373596191406, "logps/rejected": -479.7125549316406, "loss": 0.4464, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.5607531070709229, "rewards/margins": 1.2057816982269287, "rewards/rejected": -2.7665348052978516, "step": 4340 }, { "epoch": 0.6403650817017518, "grad_norm": 132.33571087083718, "learning_rate": 1.7247697903653395e-07, "logits/chosen": -1.5349242687225342, "logits/rejected": -1.3145502805709839, "logps/chosen": -502.62567138671875, "logps/rejected": -641.2630004882812, "loss": 0.4946, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.8769954442977905, "rewards/margins": 1.7127166986465454, "rewards/rejected": -3.589712619781494, "step": 4350 }, { "epoch": 0.6418371853378478, "grad_norm": 172.1367132744898, "learning_rate": 1.7125654052958465e-07, "logits/chosen": -1.3708072900772095, "logits/rejected": -0.9546362161636353, "logps/chosen": -458.1553649902344, "logps/rejected": -583.6203002929688, "loss": 0.4935, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.1133432388305664, "rewards/margins": 1.4481130838394165, "rewards/rejected": -3.5614559650421143, "step": 4360 }, { "epoch": 0.6433092889739438, "grad_norm": 128.43439548785636, "learning_rate": 1.7003818174081832e-07, "logits/chosen": -1.554459810256958, "logits/rejected": -0.766741156578064, "logps/chosen": -456.7364196777344, "logps/rejected": -559.1473388671875, "loss": 0.4098, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -2.1493165493011475, "rewards/margins": 1.483247995376587, "rewards/rejected": -3.6325645446777344, "step": 4370 }, { "epoch": 0.6447813926100397, "grad_norm": 164.25367557447944, "learning_rate": 1.6882193484869067e-07, "logits/chosen": -1.3203048706054688, "logits/rejected": -1.0798929929733276, "logps/chosen": -456.9297790527344, "logps/rejected": -523.0709228515625, "loss": 0.6116, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.040025234222412, "rewards/margins": 0.8116199374198914, "rewards/rejected": -2.851644992828369, "step": 4380 }, { "epoch": 0.6462534962461357, "grad_norm": 106.43889028279479, "learning_rate": 1.676078319758796e-07, "logits/chosen": -1.5613614320755005, "logits/rejected": -1.1758073568344116, "logps/chosen": -428.8338317871094, "logps/rejected": -505.5704040527344, "loss": 0.4929, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.6768068075180054, "rewards/margins": 1.3281102180480957, "rewards/rejected": -3.0049169063568115, "step": 4390 }, { "epoch": 0.6477255998822317, "grad_norm": 77.6104492984931, "learning_rate": 1.6639590518843643e-07, "logits/chosen": -1.5984054803848267, "logits/rejected": -1.2882667779922485, "logps/chosen": -479.27886962890625, "logps/rejected": -680.8609619140625, "loss": 0.4882, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -1.5643619298934937, "rewards/margins": 1.9690698385238647, "rewards/rejected": -3.5334315299987793, "step": 4400 }, { "epoch": 0.6491977035183277, "grad_norm": 148.22327706014093, "learning_rate": 1.6518618649493932e-07, "logits/chosen": -1.2908194065093994, "logits/rejected": -1.1842042207717896, "logps/chosen": -464.754150390625, "logps/rejected": -641.6333618164062, "loss": 0.4216, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -2.0775306224823, "rewards/margins": 1.6497796773910522, "rewards/rejected": -3.7273101806640625, "step": 4410 }, { "epoch": 0.6506698071544237, "grad_norm": 90.50992151430243, "learning_rate": 1.6397870784564777e-07, "logits/chosen": -1.3347129821777344, "logits/rejected": -1.1741870641708374, "logps/chosen": -357.79046630859375, "logps/rejected": -620.353271484375, "loss": 0.5393, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.4308732748031616, "rewards/margins": 1.9854919910430908, "rewards/rejected": -3.416365385055542, "step": 4420 }, { "epoch": 0.6521419107905196, "grad_norm": 133.53822255398748, "learning_rate": 1.6277350113165887e-07, "logits/chosen": -1.4606850147247314, "logits/rejected": -1.0468692779541016, "logps/chosen": -447.333251953125, "logps/rejected": -506.942626953125, "loss": 0.5451, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.8336231708526611, "rewards/margins": 1.1235220432281494, "rewards/rejected": -2.9571454524993896, "step": 4430 }, { "epoch": 0.6536140144266156, "grad_norm": 85.62298259101672, "learning_rate": 1.615705981840646e-07, "logits/chosen": -1.5510303974151611, "logits/rejected": -1.2544745206832886, "logps/chosen": -500.2935485839844, "logps/rejected": -588.1988525390625, "loss": 0.5077, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -1.784166693687439, "rewards/margins": 1.1019845008850098, "rewards/rejected": -2.8861513137817383, "step": 4440 }, { "epoch": 0.6550861180627117, "grad_norm": 95.86145688771262, "learning_rate": 1.6037003077311178e-07, "logits/chosen": -1.5594561100006104, "logits/rejected": -1.3050320148468018, "logps/chosen": -387.09991455078125, "logps/rejected": -573.3997802734375, "loss": 0.5525, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.4168446063995361, "rewards/margins": 1.6031293869018555, "rewards/rejected": -3.0199742317199707, "step": 4450 }, { "epoch": 0.6565582216988076, "grad_norm": 49.462407570429335, "learning_rate": 1.591718306073625e-07, "logits/chosen": -1.7461349964141846, "logits/rejected": -1.0915542840957642, "logps/chosen": -622.7125854492188, "logps/rejected": -599.0344848632812, "loss": 0.4926, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.8506015539169312, "rewards/margins": 1.1055147647857666, "rewards/rejected": -2.956116199493408, "step": 4460 }, { "epoch": 0.6580303253349036, "grad_norm": 72.30120577201066, "learning_rate": 1.5797602933285672e-07, "logits/chosen": -1.4067375659942627, "logits/rejected": -0.7826912999153137, "logps/chosen": -453.6651916503906, "logps/rejected": -578.394287109375, "loss": 0.4545, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.7367359399795532, "rewards/margins": 1.6170374155044556, "rewards/rejected": -3.3537731170654297, "step": 4470 }, { "epoch": 0.6595024289709995, "grad_norm": 103.57322200861036, "learning_rate": 1.5678265853227679e-07, "logits/chosen": -1.2267433404922485, "logits/rejected": -1.1783136129379272, "logps/chosen": -474.3990173339844, "logps/rejected": -685.5614013671875, "loss": 0.4714, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.777988076210022, "rewards/margins": 1.7630653381347656, "rewards/rejected": -3.541053295135498, "step": 4480 }, { "epoch": 0.6609745326070955, "grad_norm": 191.72471270365537, "learning_rate": 1.5559174972411271e-07, "logits/chosen": -1.4727813005447388, "logits/rejected": -0.9439153671264648, "logps/chosen": -529.7391357421875, "logps/rejected": -677.0784912109375, "loss": 0.4304, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.958915114402771, "rewards/margins": 1.8345258235931396, "rewards/rejected": -3.793440341949463, "step": 4490 }, { "epoch": 0.6624466362431916, "grad_norm": 172.11356470969972, "learning_rate": 1.5440333436183022e-07, "logits/chosen": -1.05788254737854, "logits/rejected": -0.7720486521720886, "logps/chosen": -381.38128662109375, "logps/rejected": -580.8397216796875, "loss": 0.506, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.575581669807434, "rewards/margins": 2.198709011077881, "rewards/rejected": -3.7742908000946045, "step": 4500 }, { "epoch": 0.6639187398792875, "grad_norm": 67.77998755873953, "learning_rate": 1.532174438330399e-07, "logits/chosen": -1.2263076305389404, "logits/rejected": -0.8507343530654907, "logps/chosen": -513.4577026367188, "logps/rejected": -536.4158325195312, "loss": 0.3991, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.8174861669540405, "rewards/margins": 1.1229370832443237, "rewards/rejected": -2.9404232501983643, "step": 4510 }, { "epoch": 0.6653908435153835, "grad_norm": 64.31113907340405, "learning_rate": 1.5203410945866807e-07, "logits/chosen": -1.3456099033355713, "logits/rejected": -1.0697879791259766, "logps/chosen": -442.7084045410156, "logps/rejected": -588.1572875976562, "loss": 0.4759, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.9251432418823242, "rewards/margins": 1.2387527227401733, "rewards/rejected": -3.163896322250366, "step": 4520 }, { "epoch": 0.6668629471514794, "grad_norm": 171.34155081331926, "learning_rate": 1.5085336249212982e-07, "logits/chosen": -0.9265859723091125, "logits/rejected": -0.7449873685836792, "logps/chosen": -496.591796875, "logps/rejected": -637.4634399414062, "loss": 0.5019, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.263828754425049, "rewards/margins": 1.768465280532837, "rewards/rejected": -4.032293319702148, "step": 4530 }, { "epoch": 0.6683350507875755, "grad_norm": 65.3982082721759, "learning_rate": 1.4967523411850314e-07, "logits/chosen": -1.4070160388946533, "logits/rejected": -1.2556936740875244, "logps/chosen": -415.35693359375, "logps/rejected": -572.4760131835938, "loss": 0.4425, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.8644262552261353, "rewards/margins": 1.3102327585220337, "rewards/rejected": -3.174659013748169, "step": 4540 }, { "epoch": 0.6698071544236714, "grad_norm": 86.57165747995961, "learning_rate": 1.484997554537057e-07, "logits/chosen": -1.4679739475250244, "logits/rejected": -1.3335583209991455, "logps/chosen": -442.01025390625, "logps/rejected": -588.5447998046875, "loss": 0.5407, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -1.6064382791519165, "rewards/margins": 1.1679922342300415, "rewards/rejected": -2.774430751800537, "step": 4550 }, { "epoch": 0.6712792580597674, "grad_norm": 169.8394168938232, "learning_rate": 1.4732695754367287e-07, "logits/chosen": -1.4626400470733643, "logits/rejected": -1.1115871667861938, "logps/chosen": -470.23712158203125, "logps/rejected": -604.0081787109375, "loss": 0.5613, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.7982364892959595, "rewards/margins": 1.3922786712646484, "rewards/rejected": -3.1905152797698975, "step": 4560 }, { "epoch": 0.6727513616958634, "grad_norm": 118.94477345887874, "learning_rate": 1.4615687136353787e-07, "logits/chosen": -1.449825644493103, "logits/rejected": -0.9356663823127747, "logps/chosen": -477.78485107421875, "logps/rejected": -499.4154357910156, "loss": 0.5569, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -1.8658859729766846, "rewards/margins": 0.924848735332489, "rewards/rejected": -2.7907345294952393, "step": 4570 }, { "epoch": 0.6742234653319593, "grad_norm": 114.04061458733868, "learning_rate": 1.4498952781681328e-07, "logits/chosen": -1.372307300567627, "logits/rejected": -1.2247451543807983, "logps/chosen": -397.2774963378906, "logps/rejected": -504.745849609375, "loss": 0.4547, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.627138376235962, "rewards/margins": 1.314136266708374, "rewards/rejected": -2.941274881362915, "step": 4580 }, { "epoch": 0.6756955689680554, "grad_norm": 96.84533699255098, "learning_rate": 1.4382495773457544e-07, "logits/chosen": -1.3611235618591309, "logits/rejected": -1.2581226825714111, "logps/chosen": -409.7996520996094, "logps/rejected": -437.6298828125, "loss": 0.6343, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.4521945714950562, "rewards/margins": 0.7812715768814087, "rewards/rejected": -2.233466148376465, "step": 4590 }, { "epoch": 0.6771676726041513, "grad_norm": 65.63651763622458, "learning_rate": 1.4266319187464965e-07, "logits/chosen": -1.4840503931045532, "logits/rejected": -1.2432677745819092, "logps/chosen": -336.3072204589844, "logps/rejected": -436.96893310546875, "loss": 0.4133, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.274876356124878, "rewards/margins": 1.0072202682495117, "rewards/rejected": -2.2820963859558105, "step": 4600 }, { "epoch": 0.6786397762402473, "grad_norm": 93.43058578636344, "learning_rate": 1.415042609207981e-07, "logits/chosen": -1.6027921438217163, "logits/rejected": -1.5193991661071777, "logps/chosen": -348.9036865234375, "logps/rejected": -416.65673828125, "loss": 0.451, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.2478983402252197, "rewards/margins": 1.0321069955825806, "rewards/rejected": -2.28000545501709, "step": 4610 }, { "epoch": 0.6801118798763433, "grad_norm": 273.7130929806925, "learning_rate": 1.4034819548190936e-07, "logits/chosen": -1.3875776529312134, "logits/rejected": -1.041291356086731, "logps/chosen": -430.201904296875, "logps/rejected": -521.618896484375, "loss": 0.5041, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.8789126873016357, "rewards/margins": 1.1863256692886353, "rewards/rejected": -3.0652387142181396, "step": 4620 }, { "epoch": 0.6815839835124393, "grad_norm": 83.32445019454997, "learning_rate": 1.3919502609119004e-07, "logits/chosen": -1.5672070980072021, "logits/rejected": -1.4558014869689941, "logps/chosen": -503.7395935058594, "logps/rejected": -586.6051025390625, "loss": 0.5239, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -2.074951171875, "rewards/margins": 0.9296423196792603, "rewards/rejected": -3.0045933723449707, "step": 4630 }, { "epoch": 0.6830560871485353, "grad_norm": 363.9267674425008, "learning_rate": 1.380447832053583e-07, "logits/chosen": -1.6036059856414795, "logits/rejected": -1.3291890621185303, "logps/chosen": -404.474365234375, "logps/rejected": -542.76123046875, "loss": 0.5525, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.660895586013794, "rewards/margins": 1.3934032917022705, "rewards/rejected": -3.0542988777160645, "step": 4640 }, { "epoch": 0.6845281907846312, "grad_norm": 92.33024325861862, "learning_rate": 1.3689749720383934e-07, "logits/chosen": -1.6047083139419556, "logits/rejected": -1.4916231632232666, "logps/chosen": -397.97052001953125, "logps/rejected": -433.46014404296875, "loss": 0.4918, "rewards/accuracies": 0.7000000476837158, "rewards/chosen": -1.2873114347457886, "rewards/margins": 0.6466048359870911, "rewards/rejected": -1.9339163303375244, "step": 4650 }, { "epoch": 0.6860002944207272, "grad_norm": 210.9596609081259, "learning_rate": 1.357531983879633e-07, "logits/chosen": -1.475263237953186, "logits/rejected": -1.409371256828308, "logps/chosen": -379.2502136230469, "logps/rejected": -444.80926513671875, "loss": 0.5569, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.7458136081695557, "rewards/margins": 0.8153721690177917, "rewards/rejected": -2.561185359954834, "step": 4660 }, { "epoch": 0.6874723980568233, "grad_norm": 76.12901361111314, "learning_rate": 1.3461191698016482e-07, "logits/chosen": -1.341385841369629, "logits/rejected": -1.0819720029830933, "logps/chosen": -445.7750549316406, "logps/rejected": -526.4230346679688, "loss": 0.4708, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.8485463857650757, "rewards/margins": 1.1189892292022705, "rewards/rejected": -2.9675354957580566, "step": 4670 }, { "epoch": 0.6889445016929192, "grad_norm": 100.19094061054672, "learning_rate": 1.3347368312318475e-07, "logits/chosen": -1.662327527999878, "logits/rejected": -1.525238275527954, "logps/chosen": -466.75286865234375, "logps/rejected": -534.04541015625, "loss": 0.5664, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.3829591274261475, "rewards/margins": 0.8820352554321289, "rewards/rejected": -2.2649941444396973, "step": 4680 }, { "epoch": 0.6904166053290152, "grad_norm": 166.55294478922897, "learning_rate": 1.3233852687927415e-07, "logits/chosen": -1.6528041362762451, "logits/rejected": -1.4121334552764893, "logps/chosen": -475.67962646484375, "logps/rejected": -570.8133544921875, "loss": 0.5363, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.9025332927703857, "rewards/margins": 0.821917712688446, "rewards/rejected": -2.7244510650634766, "step": 4690 }, { "epoch": 0.6918887089651111, "grad_norm": 287.29458389480885, "learning_rate": 1.3120647822940035e-07, "logits/chosen": -1.6049671173095703, "logits/rejected": -1.3347653150558472, "logps/chosen": -402.11944580078125, "logps/rejected": -521.8661499023438, "loss": 0.4867, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.3917683362960815, "rewards/margins": 1.5385221242904663, "rewards/rejected": -2.930290460586548, "step": 4700 }, { "epoch": 0.6933608126012071, "grad_norm": 101.51018229901935, "learning_rate": 1.3007756707245488e-07, "logits/chosen": -1.4593899250030518, "logits/rejected": -1.464545488357544, "logps/chosen": -394.59124755859375, "logps/rejected": -506.69622802734375, "loss": 0.5705, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.6084076166152954, "rewards/margins": 1.1785833835601807, "rewards/rejected": -2.7869908809661865, "step": 4710 }, { "epoch": 0.6948329162373031, "grad_norm": 117.5415318646873, "learning_rate": 1.2895182322446415e-07, "logits/chosen": -1.433753252029419, "logits/rejected": -1.2077696323394775, "logps/chosen": -442.948974609375, "logps/rejected": -500.4745178222656, "loss": 0.4716, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.609473466873169, "rewards/margins": 1.194166660308838, "rewards/rejected": -2.803640127182007, "step": 4720 }, { "epoch": 0.6963050198733991, "grad_norm": 122.46337705322347, "learning_rate": 1.2782927641780167e-07, "logits/chosen": -1.6359031200408936, "logits/rejected": -1.5747005939483643, "logps/chosen": -489.8575744628906, "logps/rejected": -603.5863037109375, "loss": 0.5277, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.7673225402832031, "rewards/margins": 1.133762001991272, "rewards/rejected": -2.9010844230651855, "step": 4730 }, { "epoch": 0.6977771235094951, "grad_norm": 151.42316962916956, "learning_rate": 1.2670995630040288e-07, "logits/chosen": -1.5032744407653809, "logits/rejected": -1.2521288394927979, "logps/chosen": -407.8828430175781, "logps/rejected": -499.286865234375, "loss": 0.5647, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.746633529663086, "rewards/margins": 1.0992298126220703, "rewards/rejected": -2.8458633422851562, "step": 4740 }, { "epoch": 0.699249227145591, "grad_norm": 61.373076329510646, "learning_rate": 1.2559389243498213e-07, "logits/chosen": -1.3075616359710693, "logits/rejected": -1.3392479419708252, "logps/chosen": -486.76788330078125, "logps/rejected": -608.127197265625, "loss": 0.4639, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.6814171075820923, "rewards/margins": 1.5302196741104126, "rewards/rejected": -3.211636781692505, "step": 4750 }, { "epoch": 0.7007213307816871, "grad_norm": 102.80659546722374, "learning_rate": 1.2448111429825198e-07, "logits/chosen": -1.4003568887710571, "logits/rejected": -1.1918909549713135, "logps/chosen": -527.3383178710938, "logps/rejected": -676.6248779296875, "loss": 0.4414, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -2.2130393981933594, "rewards/margins": 1.7717669010162354, "rewards/rejected": -3.984806537628174, "step": 4760 }, { "epoch": 0.702193434417783, "grad_norm": 130.0998503492656, "learning_rate": 1.2337165128014443e-07, "logits/chosen": -1.5202968120574951, "logits/rejected": -1.3023194074630737, "logps/chosen": -474.6209411621094, "logps/rejected": -554.12255859375, "loss": 0.5071, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.027803897857666, "rewards/margins": 0.8622902631759644, "rewards/rejected": -2.89009428024292, "step": 4770 }, { "epoch": 0.703665538053879, "grad_norm": 250.14239845087963, "learning_rate": 1.2226553268303494e-07, "logits/chosen": -1.0853878259658813, "logits/rejected": -0.8067598342895508, "logps/chosen": -407.2760925292969, "logps/rejected": -513.2825927734375, "loss": 0.5553, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.9570262432098389, "rewards/margins": 0.8497644662857056, "rewards/rejected": -2.806790590286255, "step": 4780 }, { "epoch": 0.705137641689975, "grad_norm": 99.66967115088167, "learning_rate": 1.2116278772096835e-07, "logits/chosen": -1.4983808994293213, "logits/rejected": -1.0246336460113525, "logps/chosen": -513.481689453125, "logps/rejected": -591.3770751953125, "loss": 0.4745, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -1.9119459390640259, "rewards/margins": 1.4465692043304443, "rewards/rejected": -3.3585147857666016, "step": 4790 }, { "epoch": 0.7066097453260709, "grad_norm": 153.93818615763, "learning_rate": 1.2006344551888736e-07, "logits/chosen": -1.3309199810028076, "logits/rejected": -1.3016533851623535, "logps/chosen": -341.87969970703125, "logps/rejected": -475.46685791015625, "loss": 0.4456, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.5952895879745483, "rewards/margins": 1.216280221939087, "rewards/rejected": -2.811569929122925, "step": 4800 }, { "epoch": 0.708081848962167, "grad_norm": 128.18739766292526, "learning_rate": 1.1896753511186364e-07, "logits/chosen": -1.6554391384124756, "logits/rejected": -1.2733652591705322, "logps/chosen": -459.8006896972656, "logps/rejected": -439.5716857910156, "loss": 0.4512, "rewards/accuracies": 0.76666659116745, "rewards/chosen": -1.749202013015747, "rewards/margins": 0.8417350053787231, "rewards/rejected": -2.5909371376037598, "step": 4810 }, { "epoch": 0.7095539525982629, "grad_norm": 113.03450199691848, "learning_rate": 1.1787508544433002e-07, "logits/chosen": -1.3578002452850342, "logits/rejected": -1.318065881729126, "logps/chosen": -434.5751037597656, "logps/rejected": -565.3150024414062, "loss": 0.5248, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.9271783828735352, "rewards/margins": 1.20933997631073, "rewards/rejected": -3.1365184783935547, "step": 4820 }, { "epoch": 0.7110260562343589, "grad_norm": 71.73553787506627, "learning_rate": 1.1678612536931718e-07, "logits/chosen": -1.4692186117172241, "logits/rejected": -1.2010374069213867, "logps/chosen": -490.8218688964844, "logps/rejected": -612.7962646484375, "loss": 0.493, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.8638207912445068, "rewards/margins": 1.580187439918518, "rewards/rejected": -3.4440078735351562, "step": 4830 }, { "epoch": 0.7124981598704548, "grad_norm": 161.8994182808025, "learning_rate": 1.1570068364769081e-07, "logits/chosen": -1.3370743989944458, "logits/rejected": -1.150339961051941, "logps/chosen": -488.45806884765625, "logps/rejected": -614.4892578125, "loss": 0.543, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -2.3346831798553467, "rewards/margins": 0.9407814145088196, "rewards/rejected": -3.2754645347595215, "step": 4840 }, { "epoch": 0.7139702635065509, "grad_norm": 124.32759682385473, "learning_rate": 1.146187889473924e-07, "logits/chosen": -1.3396797180175781, "logits/rejected": -1.2491532564163208, "logps/chosen": -529.1656494140625, "logps/rejected": -625.8582763671875, "loss": 0.6059, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.514726400375366, "rewards/margins": 1.1652178764343262, "rewards/rejected": -3.679943799972534, "step": 4850 }, { "epoch": 0.7154423671426469, "grad_norm": 106.6261258046526, "learning_rate": 1.135404698426819e-07, "logits/chosen": -1.412630319595337, "logits/rejected": -1.2308682203292847, "logps/chosen": -440.5975036621094, "logps/rejected": -562.03125, "loss": 0.4946, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.9255508184432983, "rewards/margins": 1.1983132362365723, "rewards/rejected": -3.123863697052002, "step": 4860 }, { "epoch": 0.7169144707787428, "grad_norm": 216.01010311040795, "learning_rate": 1.1246575481338305e-07, "logits/chosen": -1.7150405645370483, "logits/rejected": -1.3829970359802246, "logps/chosen": -496.76922607421875, "logps/rejected": -608.0321044921875, "loss": 0.4743, "rewards/accuracies": 0.76666659116745, "rewards/chosen": -1.8532264232635498, "rewards/margins": 1.6127201318740845, "rewards/rejected": -3.465946674346924, "step": 4870 }, { "epoch": 0.7183865744148388, "grad_norm": 150.98279648882178, "learning_rate": 1.1139467224413132e-07, "logits/chosen": -1.4484059810638428, "logits/rejected": -1.1759675741195679, "logps/chosen": -431.6138610839844, "logps/rejected": -513.0101928710938, "loss": 0.5464, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.9580078125, "rewards/margins": 1.0588047504425049, "rewards/rejected": -3.016812562942505, "step": 4880 }, { "epoch": 0.7198586780509347, "grad_norm": 72.0822214592545, "learning_rate": 1.1032725042362393e-07, "logits/chosen": -1.4815934896469116, "logits/rejected": -1.1622182130813599, "logps/chosen": -507.9613342285156, "logps/rejected": -515.327880859375, "loss": 0.5333, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -2.3548049926757812, "rewards/margins": 0.4555516839027405, "rewards/rejected": -2.810356855392456, "step": 4890 }, { "epoch": 0.7213307816870308, "grad_norm": 167.78987587220286, "learning_rate": 1.0926351754387336e-07, "logits/chosen": -1.4982646703720093, "logits/rejected": -1.4483789205551147, "logps/chosen": -490.54193115234375, "logps/rejected": -559.3733520507812, "loss": 0.4414, "rewards/accuracies": 0.7333332300186157, "rewards/chosen": -1.8408492803573608, "rewards/margins": 1.072332739830017, "rewards/rejected": -2.913182020187378, "step": 4900 }, { "epoch": 0.7228028853231268, "grad_norm": 62.624564155928404, "learning_rate": 1.0820350169946174e-07, "logits/chosen": -1.639958143234253, "logits/rejected": -1.617291808128357, "logps/chosen": -365.8565979003906, "logps/rejected": -504.843994140625, "loss": 0.4516, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.3386435508728027, "rewards/margins": 1.232550859451294, "rewards/rejected": -2.5711944103240967, "step": 4910 }, { "epoch": 0.7242749889592227, "grad_norm": 155.8678377271416, "learning_rate": 1.0714723088679983e-07, "logits/chosen": -1.6298201084136963, "logits/rejected": -1.1968131065368652, "logps/chosen": -499.95220947265625, "logps/rejected": -512.9623413085938, "loss": 0.4906, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.9635684490203857, "rewards/margins": 1.0296690464019775, "rewards/rejected": -2.993237018585205, "step": 4920 }, { "epoch": 0.7257470925953187, "grad_norm": 49.24488724479236, "learning_rate": 1.06094733003387e-07, "logits/chosen": -1.343354344367981, "logits/rejected": -1.3403180837631226, "logps/chosen": -459.63055419921875, "logps/rejected": -532.32568359375, "loss": 0.4709, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.4064701795578003, "rewards/margins": 1.1344773769378662, "rewards/rejected": -2.540947675704956, "step": 4930 }, { "epoch": 0.7272191962314147, "grad_norm": 188.84728835611227, "learning_rate": 1.0504603584707463e-07, "logits/chosen": -1.3937816619873047, "logits/rejected": -1.1371171474456787, "logps/chosen": -401.5303649902344, "logps/rejected": -485.3896484375, "loss": 0.5245, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.8510379791259766, "rewards/margins": 0.8995128870010376, "rewards/rejected": -2.7505507469177246, "step": 4940 }, { "epoch": 0.7286912998675107, "grad_norm": 209.5458787823206, "learning_rate": 1.0400116711533217e-07, "logits/chosen": -1.495514988899231, "logits/rejected": -1.0371195077896118, "logps/chosen": -507.47802734375, "logps/rejected": -536.724853515625, "loss": 0.4967, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.639105200767517, "rewards/margins": 0.920970618724823, "rewards/rejected": -2.5600757598876953, "step": 4950 }, { "epoch": 0.7301634035036066, "grad_norm": 195.11625946629115, "learning_rate": 1.029601544045148e-07, "logits/chosen": -1.2054619789123535, "logits/rejected": -1.0448256731033325, "logps/chosen": -476.5043029785156, "logps/rejected": -516.1734619140625, "loss": 0.4496, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.9478108882904053, "rewards/margins": 0.9085144996643066, "rewards/rejected": -2.856325149536133, "step": 4960 }, { "epoch": 0.7316355071397026, "grad_norm": 94.34201740908914, "learning_rate": 1.0192302520913563e-07, "logits/chosen": -1.1716625690460205, "logits/rejected": -0.9892371892929077, "logps/chosen": -438.70098876953125, "logps/rejected": -559.1282958984375, "loss": 0.5305, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.879315972328186, "rewards/margins": 1.1854360103607178, "rewards/rejected": -3.0647521018981934, "step": 4970 }, { "epoch": 0.7331076107757987, "grad_norm": 109.51344278681661, "learning_rate": 1.0088980692113872e-07, "logits/chosen": -1.1864292621612549, "logits/rejected": -0.9743377566337585, "logps/chosen": -546.6310424804688, "logps/rejected": -661.9254150390625, "loss": 0.4117, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -2.062819004058838, "rewards/margins": 1.8060537576675415, "rewards/rejected": -3.868872880935669, "step": 4980 }, { "epoch": 0.7345797144118946, "grad_norm": 96.67830014928222, "learning_rate": 9.986052682917611e-08, "logits/chosen": -1.5318454504013062, "logits/rejected": -1.413427710533142, "logps/chosen": -524.6651611328125, "logps/rejected": -549.9921264648438, "loss": 0.6576, "rewards/accuracies": 0.46666663885116577, "rewards/chosen": -2.155850410461426, "rewards/margins": 0.2569471001625061, "rewards/rejected": -2.412797451019287, "step": 4990 }, { "epoch": 0.7360518180479906, "grad_norm": 98.00148679324755, "learning_rate": 9.883521211788682e-08, "logits/chosen": -1.15413498878479, "logits/rejected": -0.798266589641571, "logps/chosen": -485.17510986328125, "logps/rejected": -548.4317626953125, "loss": 0.4739, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.8277232646942139, "rewards/margins": 1.161808967590332, "rewards/rejected": -2.989532470703125, "step": 5000 }, { "epoch": 0.7375239216840865, "grad_norm": 71.61220605596463, "learning_rate": 9.781388986717898e-08, "logits/chosen": -1.2805821895599365, "logits/rejected": -1.1228892803192139, "logps/chosen": -395.42779541015625, "logps/rejected": -517.1595458984375, "loss": 0.4585, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.844408392906189, "rewards/margins": 1.562734603881836, "rewards/rejected": -3.4071431159973145, "step": 5010 }, { "epoch": 0.7389960253201825, "grad_norm": 163.55873288904874, "learning_rate": 9.679658705151461e-08, "logits/chosen": -1.1837106943130493, "logits/rejected": -0.9553543925285339, "logps/chosen": -443.3148498535156, "logps/rejected": -583.685791015625, "loss": 0.5279, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.0693347454071045, "rewards/margins": 1.5542361736297607, "rewards/rejected": -3.6235709190368652, "step": 5020 }, { "epoch": 0.7404681289562786, "grad_norm": 70.4876460256788, "learning_rate": 9.578333053919704e-08, "logits/chosen": -1.2908295392990112, "logits/rejected": -1.153931736946106, "logps/chosen": -418.4418029785156, "logps/rejected": -645.3157348632812, "loss": 0.5578, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.9433046579360962, "rewards/margins": 1.7003980875015259, "rewards/rejected": -3.643702745437622, "step": 5030 }, { "epoch": 0.7419402325923745, "grad_norm": 67.46213633881179, "learning_rate": 9.477414709166182e-08, "logits/chosen": -1.42068350315094, "logits/rejected": -1.0340807437896729, "logps/chosen": -503.3772888183594, "logps/rejected": -612.3836059570312, "loss": 0.5481, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.9878009557724, "rewards/margins": 1.3287580013275146, "rewards/rejected": -3.316558837890625, "step": 5040 }, { "epoch": 0.7434123362284705, "grad_norm": 72.50386150719055, "learning_rate": 9.376906336276894e-08, "logits/chosen": -1.139035940170288, "logits/rejected": -0.9916818737983704, "logps/chosen": -436.18255615234375, "logps/rejected": -576.2352905273438, "loss": 0.4961, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.7689425945281982, "rewards/margins": 1.1512548923492432, "rewards/rejected": -2.9201972484588623, "step": 5050 }, { "epoch": 0.7448844398645664, "grad_norm": 97.50830907955964, "learning_rate": 9.276810589809978e-08, "logits/chosen": -1.1447765827178955, "logits/rejected": -1.1658234596252441, "logps/chosen": -352.49749755859375, "logps/rejected": -486.12042236328125, "loss": 0.496, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -1.6179523468017578, "rewards/margins": 1.1843721866607666, "rewards/rejected": -2.8023242950439453, "step": 5060 }, { "epoch": 0.7463565435006625, "grad_norm": 333.1589456993549, "learning_rate": 9.177130113425562e-08, "logits/chosen": -1.1440602540969849, "logits/rejected": -0.9470571279525757, "logps/chosen": -411.8109436035156, "logps/rejected": -523.9204711914062, "loss": 0.5742, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.2491374015808105, "rewards/margins": 1.1999796628952026, "rewards/rejected": -3.4491169452667236, "step": 5070 }, { "epoch": 0.7478286471367585, "grad_norm": 93.65396468667781, "learning_rate": 9.077867539815948e-08, "logits/chosen": -1.602186918258667, "logits/rejected": -1.2702652215957642, "logps/chosen": -421.89666748046875, "logps/rejected": -470.13934326171875, "loss": 0.4822, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.5477886199951172, "rewards/margins": 1.0641921758651733, "rewards/rejected": -2.61198091506958, "step": 5080 }, { "epoch": 0.7493007507728544, "grad_norm": 34.029670357884406, "learning_rate": 8.979025490636064e-08, "logits/chosen": -1.3914169073104858, "logits/rejected": -1.2724124193191528, "logps/chosen": -453.03094482421875, "logps/rejected": -518.9938354492188, "loss": 0.4907, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.923095703125, "rewards/margins": 0.8033218383789062, "rewards/rejected": -2.7264175415039062, "step": 5090 }, { "epoch": 0.7507728544089504, "grad_norm": 131.6843235646593, "learning_rate": 8.880606576434249e-08, "logits/chosen": -1.5269224643707275, "logits/rejected": -1.27559232711792, "logps/chosen": -420.00054931640625, "logps/rejected": -526.8175659179688, "loss": 0.476, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.71442449092865, "rewards/margins": 1.1878302097320557, "rewards/rejected": -2.902254819869995, "step": 5100 }, { "epoch": 0.7522449580450463, "grad_norm": 63.167878671951094, "learning_rate": 8.782613396583285e-08, "logits/chosen": -1.3464393615722656, "logits/rejected": -1.1066701412200928, "logps/chosen": -465.05133056640625, "logps/rejected": -484.81903076171875, "loss": 0.5136, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.8139597177505493, "rewards/margins": 0.6805374622344971, "rewards/rejected": -2.494497537612915, "step": 5110 }, { "epoch": 0.7537170616811424, "grad_norm": 64.74407065545697, "learning_rate": 8.685048539211745e-08, "logits/chosen": -1.1407724618911743, "logits/rejected": -0.9420990943908691, "logps/chosen": -507.9645080566406, "logps/rejected": -594.517578125, "loss": 0.4208, "rewards/accuracies": 0.8000000715255737, "rewards/chosen": -2.1703224182128906, "rewards/margins": 1.167057752609253, "rewards/rejected": -3.3373801708221436, "step": 5120 }, { "epoch": 0.7551891653172383, "grad_norm": 96.23348450769105, "learning_rate": 8.587914581135672e-08, "logits/chosen": -1.0492364168167114, "logits/rejected": -0.8715299367904663, "logps/chosen": -418.671875, "logps/rejected": -543.795654296875, "loss": 0.4701, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.6521028280258179, "rewards/margins": 1.2567085027694702, "rewards/rejected": -2.908811330795288, "step": 5130 }, { "epoch": 0.7566612689533343, "grad_norm": 241.03642964586655, "learning_rate": 8.491214087790447e-08, "logits/chosen": -1.5153532028198242, "logits/rejected": -1.3691776990890503, "logps/chosen": -522.0064697265625, "logps/rejected": -659.2424926757812, "loss": 0.5058, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.9308795928955078, "rewards/margins": 1.337421178817749, "rewards/rejected": -3.2683005332946777, "step": 5140 }, { "epoch": 0.7581333725894303, "grad_norm": 117.21666272567893, "learning_rate": 8.394949613163111e-08, "logits/chosen": -1.1695173978805542, "logits/rejected": -0.9150673151016235, "logps/chosen": -395.46014404296875, "logps/rejected": -482.9546813964844, "loss": 0.4571, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.7090438604354858, "rewards/margins": 1.1303540468215942, "rewards/rejected": -2.83939790725708, "step": 5150 }, { "epoch": 0.7596054762255263, "grad_norm": 49.60963349801694, "learning_rate": 8.299123699724864e-08, "logits/chosen": -1.254896879196167, "logits/rejected": -0.8868924975395203, "logps/chosen": -533.5479125976562, "logps/rejected": -600.2605590820312, "loss": 0.422, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -1.91446053981781, "rewards/margins": 1.524327278137207, "rewards/rejected": -3.4387879371643066, "step": 5160 }, { "epoch": 0.7610775798616223, "grad_norm": 103.10433149190409, "learning_rate": 8.203738878363933e-08, "logits/chosen": -0.762785792350769, "logits/rejected": -0.3832576870918274, "logps/chosen": -445.01165771484375, "logps/rejected": -505.7132873535156, "loss": 0.5257, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.137202262878418, "rewards/margins": 0.9919196963310242, "rewards/rejected": -3.129122257232666, "step": 5170 }, { "epoch": 0.7625496834977182, "grad_norm": 138.35907136125502, "learning_rate": 8.108797668318743e-08, "logits/chosen": -1.0560953617095947, "logits/rejected": -0.9559084177017212, "logps/chosen": -432.0171813964844, "logps/rejected": -558.3382568359375, "loss": 0.5275, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.146282434463501, "rewards/margins": 1.1481616497039795, "rewards/rejected": -3.2944445610046387, "step": 5180 }, { "epoch": 0.7640217871338142, "grad_norm": 116.77510339573286, "learning_rate": 8.01430257711132e-08, "logits/chosen": -1.0193790197372437, "logits/rejected": -0.8598148226737976, "logps/chosen": -467.6903381347656, "logps/rejected": -596.7313232421875, "loss": 0.4548, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -2.212892532348633, "rewards/margins": 1.633653998374939, "rewards/rejected": -3.8465466499328613, "step": 5190 }, { "epoch": 0.7654938907699103, "grad_norm": 229.39103812968878, "learning_rate": 7.92025610048114e-08, "logits/chosen": -1.185673475265503, "logits/rejected": -0.8508531451225281, "logps/chosen": -414.93475341796875, "logps/rejected": -511.0469665527344, "loss": 0.4651, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.8168270587921143, "rewards/margins": 1.3206034898757935, "rewards/rejected": -3.13742995262146, "step": 5200 }, { "epoch": 0.7669659944060062, "grad_norm": 145.52216912483013, "learning_rate": 7.826660722319165e-08, "logits/chosen": -1.303210735321045, "logits/rejected": -0.8876203298568726, "logps/chosen": -541.0342407226562, "logps/rejected": -581.6346435546875, "loss": 0.4307, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.7996604442596436, "rewards/margins": 1.1814128160476685, "rewards/rejected": -2.9810729026794434, "step": 5210 }, { "epoch": 0.7684380980421022, "grad_norm": 108.17236548702299, "learning_rate": 7.733518914602252e-08, "logits/chosen": -1.5311328172683716, "logits/rejected": -1.072148323059082, "logps/chosen": -500.82989501953125, "logps/rejected": -615.4925537109375, "loss": 0.4553, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -1.6734565496444702, "rewards/margins": 1.7444572448730469, "rewards/rejected": -3.4179139137268066, "step": 5220 }, { "epoch": 0.7699102016781981, "grad_norm": 70.65315062546658, "learning_rate": 7.640833137327882e-08, "logits/chosen": -1.0662109851837158, "logits/rejected": -0.9030225872993469, "logps/chosen": -433.59979248046875, "logps/rejected": -546.4560546875, "loss": 0.3907, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.6911451816558838, "rewards/margins": 1.4692209959030151, "rewards/rejected": -3.1603662967681885, "step": 5230 }, { "epoch": 0.7713823053142941, "grad_norm": 180.98709386648449, "learning_rate": 7.548605838449151e-08, "logits/chosen": -0.8809256553649902, "logits/rejected": -0.9900395274162292, "logps/chosen": -410.05352783203125, "logps/rejected": -558.8440551757812, "loss": 0.4325, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.7245655059814453, "rewards/margins": 1.144273042678833, "rewards/rejected": -2.8688387870788574, "step": 5240 }, { "epoch": 0.7728544089503901, "grad_norm": 124.88580544358221, "learning_rate": 7.456839453810157e-08, "logits/chosen": -1.2572662830352783, "logits/rejected": -1.0659639835357666, "logps/chosen": -494.0467834472656, "logps/rejected": -647.9281005859375, "loss": 0.4469, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.1124119758605957, "rewards/margins": 1.4500975608825684, "rewards/rejected": -3.5625100135803223, "step": 5250 }, { "epoch": 0.7743265125864861, "grad_norm": 154.45317247906624, "learning_rate": 7.365536407081633e-08, "logits/chosen": -1.251163363456726, "logits/rejected": -1.229315161705017, "logps/chosen": -428.061279296875, "logps/rejected": -601.31689453125, "loss": 0.404, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.6897375583648682, "rewards/margins": 1.5642296075820923, "rewards/rejected": -3.253966808319092, "step": 5260 }, { "epoch": 0.7757986162225821, "grad_norm": 85.77593336032358, "learning_rate": 7.274699109696975e-08, "logits/chosen": -1.0087244510650635, "logits/rejected": -0.6363364458084106, "logps/chosen": -390.22314453125, "logps/rejected": -438.15655517578125, "loss": 0.549, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.5335338115692139, "rewards/margins": 0.8562105298042297, "rewards/rejected": -2.389744520187378, "step": 5270 }, { "epoch": 0.777270719858678, "grad_norm": 155.2419340848915, "learning_rate": 7.184329960788491e-08, "logits/chosen": -1.0169925689697266, "logits/rejected": -0.7420837879180908, "logps/chosen": -395.13763427734375, "logps/rejected": -565.1148681640625, "loss": 0.4765, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.8920724391937256, "rewards/margins": 1.4287753105163574, "rewards/rejected": -3.320847988128662, "step": 5280 }, { "epoch": 0.7787428234947741, "grad_norm": 87.81525453347075, "learning_rate": 7.094431347124092e-08, "logits/chosen": -0.8687102198600769, "logits/rejected": -0.6938323974609375, "logps/chosen": -404.7712707519531, "logps/rejected": -538.3594970703125, "loss": 0.3722, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.5631678104400635, "rewards/margins": 1.6385186910629272, "rewards/rejected": -3.201686382293701, "step": 5290 }, { "epoch": 0.78021492713087, "grad_norm": 59.173002113288824, "learning_rate": 7.00500564304424e-08, "logits/chosen": -1.2083094120025635, "logits/rejected": -0.9714385271072388, "logps/chosen": -557.9366455078125, "logps/rejected": -621.3553466796875, "loss": 0.4425, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.9470123052597046, "rewards/margins": 1.1327309608459473, "rewards/rejected": -3.0797431468963623, "step": 5300 }, { "epoch": 0.781687030766966, "grad_norm": 215.11128920992735, "learning_rate": 6.916055210399219e-08, "logits/chosen": -1.3841431140899658, "logits/rejected": -1.386099100112915, "logps/chosen": -473.8697814941406, "logps/rejected": -637.9520263671875, "loss": 0.539, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.7102562189102173, "rewards/margins": 1.2155544757843018, "rewards/rejected": -2.9258105754852295, "step": 5310 }, { "epoch": 0.783159134403062, "grad_norm": 225.67677440514703, "learning_rate": 6.827582398486797e-08, "logits/chosen": -1.2445056438446045, "logits/rejected": -0.9456964731216431, "logps/chosen": -578.6286010742188, "logps/rejected": -627.7783203125, "loss": 0.6115, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -1.9115711450576782, "rewards/margins": 1.503936529159546, "rewards/rejected": -3.4155075550079346, "step": 5320 }, { "epoch": 0.7846312380391579, "grad_norm": 77.17729472372167, "learning_rate": 6.739589543990118e-08, "logits/chosen": -0.9577809572219849, "logits/rejected": -0.5505359768867493, "logps/chosen": -462.0064392089844, "logps/rejected": -527.6776123046875, "loss": 0.5602, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.7571370601654053, "rewards/margins": 1.3212145566940308, "rewards/rejected": -3.0783514976501465, "step": 5330 }, { "epoch": 0.786103341675254, "grad_norm": 109.22908357628783, "learning_rate": 6.652078970916037e-08, "logits/chosen": -1.3481218814849854, "logits/rejected": -1.0555284023284912, "logps/chosen": -458.81292724609375, "logps/rejected": -632.86572265625, "loss": 0.45, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.9127309322357178, "rewards/margins": 1.7171541452407837, "rewards/rejected": -3.629885196685791, "step": 5340 }, { "epoch": 0.7875754453113499, "grad_norm": 53.024674570795995, "learning_rate": 6.565052990533715e-08, "logits/chosen": -1.2405158281326294, "logits/rejected": -1.1840492486953735, "logps/chosen": -474.354248046875, "logps/rejected": -581.2073364257812, "loss": 0.494, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.223118305206299, "rewards/margins": 1.165349006652832, "rewards/rejected": -3.388467311859131, "step": 5350 }, { "epoch": 0.7890475489474459, "grad_norm": 238.66915414797674, "learning_rate": 6.478513901313615e-08, "logits/chosen": -0.9796088933944702, "logits/rejected": -0.9768195152282715, "logps/chosen": -406.1162109375, "logps/rejected": -567.86328125, "loss": 0.5971, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.611419677734375, "rewards/margins": 1.1555076837539673, "rewards/rejected": -2.766927480697632, "step": 5360 }, { "epoch": 0.7905196525835418, "grad_norm": 140.70071817008576, "learning_rate": 6.392463988866714e-08, "logits/chosen": -1.0570393800735474, "logits/rejected": -0.9775602221488953, "logps/chosen": -456.1234436035156, "logps/rejected": -606.5938110351562, "loss": 0.5794, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.8095552921295166, "rewards/margins": 1.2307498455047607, "rewards/rejected": -3.0403048992156982, "step": 5370 }, { "epoch": 0.7919917562196379, "grad_norm": 52.10965788819851, "learning_rate": 6.306905525884227e-08, "logits/chosen": -1.5277817249298096, "logits/rejected": -1.355376958847046, "logps/chosen": -475.4259338378906, "logps/rejected": -586.855224609375, "loss": 0.4918, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.0407283306121826, "rewards/margins": 1.0685203075408936, "rewards/rejected": -3.109248638153076, "step": 5380 }, { "epoch": 0.7934638598557339, "grad_norm": 175.36706272305227, "learning_rate": 6.221840772077525e-08, "logits/chosen": -1.2016507387161255, "logits/rejected": -1.0885107517242432, "logps/chosen": -439.05462646484375, "logps/rejected": -516.28515625, "loss": 0.4896, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -2.0163323879241943, "rewards/margins": 1.0620869398117065, "rewards/rejected": -3.0784192085266113, "step": 5390 }, { "epoch": 0.7949359634918298, "grad_norm": 105.3003864094294, "learning_rate": 6.137271974118468e-08, "logits/chosen": -1.4145647287368774, "logits/rejected": -1.1715717315673828, "logps/chosen": -466.78466796875, "logps/rejected": -536.9066162109375, "loss": 0.4988, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -2.034123659133911, "rewards/margins": 0.9236257672309875, "rewards/rejected": -2.957749366760254, "step": 5400 }, { "epoch": 0.7964080671279258, "grad_norm": 76.8887580233539, "learning_rate": 6.05320136558011e-08, "logits/chosen": -1.4233685731887817, "logits/rejected": -1.3150105476379395, "logps/chosen": -439.65411376953125, "logps/rejected": -577.3585815429688, "loss": 0.5535, "rewards/accuracies": 0.7000000476837158, "rewards/chosen": -1.8920074701309204, "rewards/margins": 1.148402452468872, "rewards/rejected": -3.040409564971924, "step": 5410 }, { "epoch": 0.7978801707640217, "grad_norm": 123.14342704766716, "learning_rate": 5.969631166877607e-08, "logits/chosen": -1.337247610092163, "logits/rejected": -0.9550000429153442, "logps/chosen": -489.42742919921875, "logps/rejected": -566.384765625, "loss": 0.4342, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.637761116027832, "rewards/margins": 1.4270638227462769, "rewards/rejected": -3.0648245811462402, "step": 5420 }, { "epoch": 0.7993522744001178, "grad_norm": 115.54034393158733, "learning_rate": 5.8865635852096754e-08, "logits/chosen": -1.2357947826385498, "logits/rejected": -1.0595812797546387, "logps/chosen": -422.2577209472656, "logps/rejected": -501.9385681152344, "loss": 0.4183, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.8146345615386963, "rewards/margins": 1.0913867950439453, "rewards/rejected": -2.9060215950012207, "step": 5430 }, { "epoch": 0.8008243780362138, "grad_norm": 157.6274293347524, "learning_rate": 5.8040008145002344e-08, "logits/chosen": -1.7033611536026, "logits/rejected": -0.9675869941711426, "logps/chosen": -513.8781127929688, "logps/rejected": -606.8839111328125, "loss": 0.414, "rewards/accuracies": 0.9000000953674316, "rewards/chosen": -1.8686834573745728, "rewards/margins": 1.5403255224227905, "rewards/rejected": -3.409008741378784, "step": 5440 }, { "epoch": 0.8022964816723097, "grad_norm": 111.81350505285386, "learning_rate": 5.721945035340511e-08, "logits/chosen": -1.1051543951034546, "logits/rejected": -0.8956894874572754, "logps/chosen": -411.0330505371094, "logps/rejected": -571.52880859375, "loss": 0.429, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.614373803138733, "rewards/margins": 1.9639537334442139, "rewards/rejected": -3.5783278942108154, "step": 5450 }, { "epoch": 0.8037685853084057, "grad_norm": 101.85766958151592, "learning_rate": 5.640398414931399e-08, "logits/chosen": -1.013607382774353, "logits/rejected": -1.0927541255950928, "logps/chosen": -410.86724853515625, "logps/rejected": -541.094482421875, "loss": 0.633, "rewards/accuracies": 0.5999999642372131, "rewards/chosen": -2.005836009979248, "rewards/margins": 0.868213951587677, "rewards/rejected": -2.874049663543701, "step": 5460 }, { "epoch": 0.8052406889445017, "grad_norm": 163.6961418271051, "learning_rate": 5.5593631070262296e-08, "logits/chosen": -1.2479948997497559, "logits/rejected": -0.8561438322067261, "logps/chosen": -516.5875244140625, "logps/rejected": -675.2960815429688, "loss": 0.4573, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -2.2040138244628906, "rewards/margins": 1.7423547506332397, "rewards/rejected": -3.946368455886841, "step": 5470 }, { "epoch": 0.8067127925805977, "grad_norm": 74.07186173313391, "learning_rate": 5.478841251873922e-08, "logits/chosen": -1.3214751482009888, "logits/rejected": -0.9202545285224915, "logps/chosen": -486.7603454589844, "logps/rejected": -577.1681518554688, "loss": 0.4266, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -1.4990867376327515, "rewards/margins": 1.670100212097168, "rewards/rejected": -3.169187068939209, "step": 5480 }, { "epoch": 0.8081848962166936, "grad_norm": 59.471481247489244, "learning_rate": 5.398834976162414e-08, "logits/chosen": -1.2711598873138428, "logits/rejected": -1.0352222919464111, "logps/chosen": -439.3692932128906, "logps/rejected": -608.0352172851562, "loss": 0.4223, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.5429414510726929, "rewards/margins": 1.9663690328598022, "rewards/rejected": -3.509310245513916, "step": 5490 }, { "epoch": 0.8096569998527896, "grad_norm": 92.59368398233123, "learning_rate": 5.319346392962545e-08, "logits/chosen": -1.1628122329711914, "logits/rejected": -1.3315460681915283, "logps/chosen": -462.98309326171875, "logps/rejected": -596.800048828125, "loss": 0.4933, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.9284751415252686, "rewards/margins": 1.2932318449020386, "rewards/rejected": -3.2217068672180176, "step": 5500 }, { "epoch": 0.8111291034888857, "grad_norm": 115.56862692289259, "learning_rate": 5.24037760167218e-08, "logits/chosen": -1.342844843864441, "logits/rejected": -1.0954813957214355, "logps/chosen": -491.49053955078125, "logps/rejected": -619.1533813476562, "loss": 0.4144, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.6697956323623657, "rewards/margins": 1.7845573425292969, "rewards/rejected": -3.454352855682373, "step": 5510 }, { "epoch": 0.8126012071249816, "grad_norm": 125.46726804948204, "learning_rate": 5.161930687960808e-08, "logits/chosen": -1.2879449129104614, "logits/rejected": -0.6370636224746704, "logps/chosen": -508.4657287597656, "logps/rejected": -519.69921875, "loss": 0.5967, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -2.0370121002197266, "rewards/margins": 1.1051788330078125, "rewards/rejected": -3.142190933227539, "step": 5520 }, { "epoch": 0.8140733107610776, "grad_norm": 179.96357276317437, "learning_rate": 5.0840077237144594e-08, "logits/chosen": -1.3536159992218018, "logits/rejected": -0.7188901901245117, "logps/chosen": -476.391357421875, "logps/rejected": -564.7723999023438, "loss": 0.4715, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.8136851787567139, "rewards/margins": 1.4985334873199463, "rewards/rejected": -3.312218427658081, "step": 5530 }, { "epoch": 0.8155454143971735, "grad_norm": 137.98162018971811, "learning_rate": 5.006610766980945e-08, "logits/chosen": -1.4704669713974, "logits/rejected": -1.1928757429122925, "logps/chosen": -478.86285400390625, "logps/rejected": -580.6500854492188, "loss": 0.5194, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.861915946006775, "rewards/margins": 1.3256458044052124, "rewards/rejected": -3.1875619888305664, "step": 5540 }, { "epoch": 0.8170175180332695, "grad_norm": 183.42102471753302, "learning_rate": 4.929741861915571e-08, "logits/chosen": -1.4357191324234009, "logits/rejected": -1.1636178493499756, "logps/chosen": -431.68670654296875, "logps/rejected": -512.5819091796875, "loss": 0.4701, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.772092580795288, "rewards/margins": 1.139080286026001, "rewards/rejected": -2.911172866821289, "step": 5550 }, { "epoch": 0.8184896216693656, "grad_norm": 198.6027603437246, "learning_rate": 4.853403038727047e-08, "logits/chosen": -1.0890223979949951, "logits/rejected": -1.047532081604004, "logps/chosen": -422.9931640625, "logps/rejected": -612.4053344726562, "loss": 0.5113, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.9905509948730469, "rewards/margins": 1.0186119079589844, "rewards/rejected": -3.009162425994873, "step": 5560 }, { "epoch": 0.8199617253054615, "grad_norm": 168.23022056516947, "learning_rate": 4.777596313623966e-08, "logits/chosen": -1.2151178121566772, "logits/rejected": -1.0620383024215698, "logps/chosen": -509.9613342285156, "logps/rejected": -609.4564208984375, "loss": 0.4881, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.6974384784698486, "rewards/margins": 1.595670461654663, "rewards/rejected": -3.293109178543091, "step": 5570 }, { "epoch": 0.8214338289415575, "grad_norm": 72.3016109109807, "learning_rate": 4.702323688761492e-08, "logits/chosen": -1.3811545372009277, "logits/rejected": -0.8901753425598145, "logps/chosen": -473.8961486816406, "logps/rejected": -604.1268310546875, "loss": 0.4628, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -2.2072179317474365, "rewards/margins": 1.3967196941375732, "rewards/rejected": -3.603937864303589, "step": 5580 }, { "epoch": 0.8229059325776534, "grad_norm": 135.1282785356735, "learning_rate": 4.627587152188522e-08, "logits/chosen": -1.446920394897461, "logits/rejected": -1.1027758121490479, "logps/chosen": -443.819091796875, "logps/rejected": -525.2467041015625, "loss": 0.4622, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.6185214519500732, "rewards/margins": 1.4930517673492432, "rewards/rejected": -3.1115729808807373, "step": 5590 }, { "epoch": 0.8243780362137495, "grad_norm": 52.554048763874356, "learning_rate": 4.5533886777951395e-08, "logits/chosen": -1.4271553754806519, "logits/rejected": -1.3268260955810547, "logps/chosen": -479.57855224609375, "logps/rejected": -691.8262939453125, "loss": 0.4729, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.5048121213912964, "rewards/margins": 1.455271601676941, "rewards/rejected": -2.960083484649658, "step": 5600 }, { "epoch": 0.8258501398498455, "grad_norm": 177.29394626551294, "learning_rate": 4.4797302252604916e-08, "logits/chosen": -1.188375473022461, "logits/rejected": -0.9038418531417847, "logps/chosen": -409.9644470214844, "logps/rejected": -406.25323486328125, "loss": 0.5234, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.814875841140747, "rewards/margins": 0.5175659656524658, "rewards/rejected": -2.332441806793213, "step": 5610 }, { "epoch": 0.8273222434859414, "grad_norm": 119.22624538847619, "learning_rate": 4.4066137400010555e-08, "logits/chosen": -1.355344295501709, "logits/rejected": -0.9832057952880859, "logps/chosen": -445.841796875, "logps/rejected": -641.5125732421875, "loss": 0.4839, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -1.7873265743255615, "rewards/margins": 2.185568332672119, "rewards/rejected": -3.9728951454162598, "step": 5620 }, { "epoch": 0.8287943471220374, "grad_norm": 137.0047831227428, "learning_rate": 4.3340411531192306e-08, "logits/chosen": -1.5107392072677612, "logits/rejected": -1.3040306568145752, "logps/chosen": -445.95208740234375, "logps/rejected": -476.7379455566406, "loss": 0.4406, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.3861254453659058, "rewards/margins": 1.0516457557678223, "rewards/rejected": -2.4377713203430176, "step": 5630 }, { "epoch": 0.8302664507581333, "grad_norm": 82.84525438977428, "learning_rate": 4.262014381352363e-08, "logits/chosen": -1.3577890396118164, "logits/rejected": -0.9183663129806519, "logps/chosen": -452.920654296875, "logps/rejected": -557.4637451171875, "loss": 0.5162, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.7773888111114502, "rewards/margins": 1.2821528911590576, "rewards/rejected": -3.0595414638519287, "step": 5640 }, { "epoch": 0.8317385543942294, "grad_norm": 76.17226865199585, "learning_rate": 4.1905353270220795e-08, "logits/chosen": -1.280076265335083, "logits/rejected": -0.932104229927063, "logps/chosen": -393.5255126953125, "logps/rejected": -562.3932495117188, "loss": 0.5413, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.365314245223999, "rewards/margins": 1.344012975692749, "rewards/rejected": -2.709327459335327, "step": 5650 }, { "epoch": 0.8332106580303253, "grad_norm": 121.3359110614011, "learning_rate": 4.119605877984089e-08, "logits/chosen": -1.3621351718902588, "logits/rejected": -1.2421245574951172, "logps/chosen": -403.90557861328125, "logps/rejected": -476.821533203125, "loss": 0.7086, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.9038448333740234, "rewards/margins": 0.5362643003463745, "rewards/rejected": -2.4401090145111084, "step": 5660 }, { "epoch": 0.8346827616664213, "grad_norm": 83.12577355392818, "learning_rate": 4.049227907578284e-08, "logits/chosen": -1.2348663806915283, "logits/rejected": -1.2635643482208252, "logps/chosen": -422.54376220703125, "logps/rejected": -676.0519409179688, "loss": 0.4528, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.659354567527771, "rewards/margins": 2.0648245811462402, "rewards/rejected": -3.7241787910461426, "step": 5670 }, { "epoch": 0.8361548653025173, "grad_norm": 116.01745966168035, "learning_rate": 3.979403274579313e-08, "logits/chosen": -1.3450870513916016, "logits/rejected": -0.9937416315078735, "logps/chosen": -479.0702209472656, "logps/rejected": -600.4904174804688, "loss": 0.4331, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.8521366119384766, "rewards/margins": 1.6752088069915771, "rewards/rejected": -3.527345657348633, "step": 5680 }, { "epoch": 0.8376269689386133, "grad_norm": 211.47659985662133, "learning_rate": 3.9101338231474354e-08, "logits/chosen": -1.4885746240615845, "logits/rejected": -1.256639838218689, "logps/chosen": -473.7394104003906, "logps/rejected": -589.0265502929688, "loss": 0.5276, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.9350303411483765, "rewards/margins": 1.2548571825027466, "rewards/rejected": -3.189887523651123, "step": 5690 }, { "epoch": 0.8390990725747093, "grad_norm": 205.77550522367355, "learning_rate": 3.841421382779827e-08, "logits/chosen": -1.3588941097259521, "logits/rejected": -1.2172741889953613, "logps/chosen": -419.08978271484375, "logps/rejected": -616.7435302734375, "loss": 0.4747, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.7490804195404053, "rewards/margins": 1.648638367652893, "rewards/rejected": -3.397718906402588, "step": 5700 }, { "epoch": 0.8405711762108052, "grad_norm": 102.47784262945792, "learning_rate": 3.773267768262289e-08, "logits/chosen": -1.27297842502594, "logits/rejected": -0.9850039482116699, "logps/chosen": -427.94305419921875, "logps/rejected": -562.6896362304688, "loss": 0.5676, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.608070731163025, "rewards/margins": 1.3251826763153076, "rewards/rejected": -2.933253526687622, "step": 5710 }, { "epoch": 0.8420432798469012, "grad_norm": 71.0823358327679, "learning_rate": 3.7056747796212844e-08, "logits/chosen": -1.2771185636520386, "logits/rejected": -0.8199060559272766, "logps/chosen": -396.62384033203125, "logps/rejected": -527.8873901367188, "loss": 0.4536, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.4784718751907349, "rewards/margins": 1.6878697872161865, "rewards/rejected": -3.166341543197632, "step": 5720 }, { "epoch": 0.8435153834829971, "grad_norm": 101.30985808697004, "learning_rate": 3.63864420207643e-08, "logits/chosen": -1.4370369911193848, "logits/rejected": -1.2566996812820435, "logps/chosen": -465.5155334472656, "logps/rejected": -630.5440673828125, "loss": 0.3811, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -1.6955715417861938, "rewards/margins": 1.8823350667953491, "rewards/rejected": -3.577906847000122, "step": 5730 }, { "epoch": 0.8449874871190932, "grad_norm": 100.84237966720134, "learning_rate": 3.5721778059933096e-08, "logits/chosen": -1.3979724645614624, "logits/rejected": -1.179774522781372, "logps/chosen": -445.75372314453125, "logps/rejected": -442.28839111328125, "loss": 0.5404, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.8010488748550415, "rewards/margins": 0.5255202651023865, "rewards/rejected": -2.3265693187713623, "step": 5740 }, { "epoch": 0.8464595907551892, "grad_norm": 60.767936064669804, "learning_rate": 3.506277346836731e-08, "logits/chosen": -1.1453638076782227, "logits/rejected": -1.4349292516708374, "logps/chosen": -452.9850158691406, "logps/rejected": -645.1492919921875, "loss": 0.5655, "rewards/accuracies": 0.5333333015441895, "rewards/chosen": -2.2502622604370117, "rewards/margins": 0.9999488592147827, "rewards/rejected": -3.250211238861084, "step": 5750 }, { "epoch": 0.8479316943912851, "grad_norm": 152.6111398448886, "learning_rate": 3.440944565124371e-08, "logits/chosen": -1.000203013420105, "logits/rejected": -0.7757139801979065, "logps/chosen": -362.6944885253906, "logps/rejected": -487.8475646972656, "loss": 0.3965, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.3980891704559326, "rewards/margins": 1.1738362312316895, "rewards/rejected": -2.571925640106201, "step": 5760 }, { "epoch": 0.8494037980273811, "grad_norm": 163.80390518535975, "learning_rate": 3.376181186380808e-08, "logits/chosen": -1.407921552658081, "logits/rejected": -1.1443045139312744, "logps/chosen": -446.7001953125, "logps/rejected": -488.558349609375, "loss": 0.5739, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -2.02390718460083, "rewards/margins": 0.6599318385124207, "rewards/rejected": -2.6838388442993164, "step": 5770 }, { "epoch": 0.8508759016634772, "grad_norm": 213.6345979004298, "learning_rate": 3.311988921091935e-08, "logits/chosen": -1.5590388774871826, "logits/rejected": -1.0573437213897705, "logps/chosen": -445.07977294921875, "logps/rejected": -486.111083984375, "loss": 0.4314, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.4946223497390747, "rewards/margins": 1.2161999940872192, "rewards/rejected": -2.710822343826294, "step": 5780 }, { "epoch": 0.8523480052995731, "grad_norm": 114.36351797505455, "learning_rate": 3.248369464659775e-08, "logits/chosen": -1.1733253002166748, "logits/rejected": -1.0773212909698486, "logps/chosen": -366.6900634765625, "logps/rejected": -502.72845458984375, "loss": 0.4712, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.5471848249435425, "rewards/margins": 1.5482925176620483, "rewards/rejected": -3.0954771041870117, "step": 5790 }, { "epoch": 0.8538201089356691, "grad_norm": 86.24945234823068, "learning_rate": 3.1853244973577306e-08, "logits/chosen": -1.417776107788086, "logits/rejected": -0.8327361941337585, "logps/chosen": -466.37066650390625, "logps/rejected": -610.36865234375, "loss": 0.5163, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.8226988315582275, "rewards/margins": 1.4830236434936523, "rewards/rejected": -3.30572247505188, "step": 5800 }, { "epoch": 0.855292212571765, "grad_norm": 82.26745113001945, "learning_rate": 3.122855684286185e-08, "logits/chosen": -1.0725051164627075, "logits/rejected": -0.935965895652771, "logps/chosen": -491.5113220214844, "logps/rejected": -638.8419189453125, "loss": 0.4563, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.990782380104065, "rewards/margins": 1.335800290107727, "rewards/rejected": -3.326582670211792, "step": 5810 }, { "epoch": 0.8567643162078611, "grad_norm": 62.483405589834824, "learning_rate": 3.060964675328545e-08, "logits/chosen": -1.1046890020370483, "logits/rejected": -0.9802217483520508, "logps/chosen": -476.57305908203125, "logps/rejected": -584.443359375, "loss": 0.5224, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.23058819770813, "rewards/margins": 0.9979446530342102, "rewards/rejected": -3.2285327911376953, "step": 5820 }, { "epoch": 0.858236419843957, "grad_norm": 130.02543143459314, "learning_rate": 2.999653105107644e-08, "logits/chosen": -1.2632063627243042, "logits/rejected": -1.0915645360946655, "logps/chosen": -381.5784912109375, "logps/rejected": -556.1746826171875, "loss": 0.5175, "rewards/accuracies": 0.9000000953674316, "rewards/chosen": -1.3862593173980713, "rewards/margins": 1.8085914850234985, "rewards/rejected": -3.1948509216308594, "step": 5830 }, { "epoch": 0.859708523480053, "grad_norm": 97.1206867972697, "learning_rate": 2.9389225929425637e-08, "logits/chosen": -1.3377158641815186, "logits/rejected": -1.2011384963989258, "logps/chosen": -396.65777587890625, "logps/rejected": -504.1952209472656, "loss": 0.4344, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.5112955570220947, "rewards/margins": 1.373363971710205, "rewards/rejected": -2.8846595287323, "step": 5840 }, { "epoch": 0.861180627116149, "grad_norm": 169.86234185913474, "learning_rate": 2.8787747428058977e-08, "logits/chosen": -1.3013274669647217, "logits/rejected": -1.3599807024002075, "logps/chosen": -475.5846252441406, "logps/rejected": -583.6883544921875, "loss": 0.4146, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.5154919624328613, "rewards/margins": 1.2250664234161377, "rewards/rejected": -2.740558385848999, "step": 5850 }, { "epoch": 0.8626527307522449, "grad_norm": 93.00186430580612, "learning_rate": 2.8192111432813554e-08, "logits/chosen": -1.704040765762329, "logits/rejected": -1.2342931032180786, "logps/chosen": -447.02362060546875, "logps/rejected": -556.9030151367188, "loss": 0.598, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.7818158864974976, "rewards/margins": 1.0600388050079346, "rewards/rejected": -2.8418548107147217, "step": 5860 }, { "epoch": 0.864124834388341, "grad_norm": 110.29006178066524, "learning_rate": 2.7602333675218443e-08, "logits/chosen": -1.392507553100586, "logits/rejected": -1.0788378715515137, "logps/chosen": -433.29779052734375, "logps/rejected": -455.02471923828125, "loss": 0.4711, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.8213212490081787, "rewards/margins": 0.8197693824768066, "rewards/rejected": -2.6410906314849854, "step": 5870 }, { "epoch": 0.8655969380244369, "grad_norm": 94.93639517672632, "learning_rate": 2.701842973207874e-08, "logits/chosen": -1.1838589906692505, "logits/rejected": -1.0877768993377686, "logps/chosen": -428.2568359375, "logps/rejected": -584.2926025390625, "loss": 0.4963, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.5862526893615723, "rewards/margins": 1.5620958805084229, "rewards/rejected": -3.148348569869995, "step": 5880 }, { "epoch": 0.8670690416605329, "grad_norm": 71.54103856862521, "learning_rate": 2.6440415025064417e-08, "logits/chosen": -1.4555714130401611, "logits/rejected": -1.2023519277572632, "logps/chosen": -458.5950622558594, "logps/rejected": -593.3828125, "loss": 0.5073, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -1.3977679014205933, "rewards/margins": 1.7654355764389038, "rewards/rejected": -3.163203477859497, "step": 5890 }, { "epoch": 0.8685411452966288, "grad_norm": 113.61398731243004, "learning_rate": 2.5868304820303027e-08, "logits/chosen": -1.3414885997772217, "logits/rejected": -1.0834671258926392, "logps/chosen": -480.46063232421875, "logps/rejected": -555.4865112304688, "loss": 0.4831, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.8580623865127563, "rewards/margins": 1.2035136222839355, "rewards/rejected": -3.0615758895874023, "step": 5900 }, { "epoch": 0.8700132489327249, "grad_norm": 169.16449158549293, "learning_rate": 2.5302114227976513e-08, "logits/chosen": -1.4471521377563477, "logits/rejected": -1.2723208665847778, "logps/chosen": -430.6087341308594, "logps/rejected": -541.4777221679688, "loss": 0.3965, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.3868787288665771, "rewards/margins": 1.3914029598236084, "rewards/rejected": -2.7782816886901855, "step": 5910 }, { "epoch": 0.8714853525688209, "grad_norm": 124.14812313891947, "learning_rate": 2.4741858201922065e-08, "logits/chosen": -1.3322300910949707, "logits/rejected": -0.9847902059555054, "logps/chosen": -454.60015869140625, "logps/rejected": -622.1671142578125, "loss": 0.465, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.8008426427841187, "rewards/margins": 1.62545907497406, "rewards/rejected": -3.4263014793395996, "step": 5920 }, { "epoch": 0.8729574562049168, "grad_norm": 59.57786300290591, "learning_rate": 2.4187551539237073e-08, "logits/chosen": -1.3692054748535156, "logits/rejected": -1.0318745374679565, "logps/chosen": -430.62115478515625, "logps/rejected": -508.8824768066406, "loss": 0.5123, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.483989953994751, "rewards/margins": 1.3129962682724, "rewards/rejected": -2.7969861030578613, "step": 5930 }, { "epoch": 0.8744295598410128, "grad_norm": 48.135899110660965, "learning_rate": 2.363920887988849e-08, "logits/chosen": -1.3396302461624146, "logits/rejected": -1.0970475673675537, "logps/chosen": -416.86474609375, "logps/rejected": -642.1658935546875, "loss": 0.436, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -1.9330217838287354, "rewards/margins": 1.992821455001831, "rewards/rejected": -3.9258437156677246, "step": 5940 }, { "epoch": 0.8759016634771087, "grad_norm": 73.33774881582387, "learning_rate": 2.3096844706326112e-08, "logits/chosen": -1.3286715745925903, "logits/rejected": -0.9419041872024536, "logps/chosen": -470.03082275390625, "logps/rejected": -600.4696655273438, "loss": 0.4364, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -1.775228500366211, "rewards/margins": 1.9548699855804443, "rewards/rejected": -3.730098247528076, "step": 5950 }, { "epoch": 0.8773737671132048, "grad_norm": 118.10841838390651, "learning_rate": 2.2560473343100155e-08, "logits/chosen": -1.3343017101287842, "logits/rejected": -0.9438940286636353, "logps/chosen": -402.0434875488281, "logps/rejected": -506.3617248535156, "loss": 0.5469, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -1.7077451944351196, "rewards/margins": 1.385524034500122, "rewards/rejected": -3.0932693481445312, "step": 5960 }, { "epoch": 0.8788458707493008, "grad_norm": 128.7716793905253, "learning_rate": 2.20301089564828e-08, "logits/chosen": -1.3156306743621826, "logits/rejected": -0.6910329461097717, "logps/chosen": -434.342529296875, "logps/rejected": -556.0278930664062, "loss": 0.3873, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.5784485340118408, "rewards/margins": 1.608205795288086, "rewards/rejected": -3.1866543292999268, "step": 5970 }, { "epoch": 0.8803179743853967, "grad_norm": 173.2362279170518, "learning_rate": 2.150576555409392e-08, "logits/chosen": -1.238092303276062, "logits/rejected": -0.8196524381637573, "logps/chosen": -345.23101806640625, "logps/rejected": -448.9268493652344, "loss": 0.5213, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.5229777097702026, "rewards/margins": 1.2405390739440918, "rewards/rejected": -2.763516664505005, "step": 5980 }, { "epoch": 0.8817900780214927, "grad_norm": 107.64309048981154, "learning_rate": 2.0987456984531472e-08, "logits/chosen": -1.6010726690292358, "logits/rejected": -1.2792376279830933, "logps/chosen": -423.9590759277344, "logps/rejected": -515.4378662109375, "loss": 0.526, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": -1.7081773281097412, "rewards/margins": 1.0969882011413574, "rewards/rejected": -2.8051657676696777, "step": 5990 }, { "epoch": 0.8832621816575887, "grad_norm": 69.37673759463547, "learning_rate": 2.0475196937005557e-08, "logits/chosen": -1.2657551765441895, "logits/rejected": -1.1628000736236572, "logps/chosen": -331.22576904296875, "logps/rejected": -553.7278442382812, "loss": 0.4878, "rewards/accuracies": 0.76666659116745, "rewards/chosen": -1.2025068998336792, "rewards/margins": 1.6287647485733032, "rewards/rejected": -2.8312716484069824, "step": 6000 }, { "epoch": 0.8847342852936847, "grad_norm": 132.04208792710588, "learning_rate": 1.9968998940976784e-08, "logits/chosen": -1.4524221420288086, "logits/rejected": -1.0347516536712646, "logps/chosen": -428.4149475097656, "logps/rejected": -584.2083740234375, "loss": 0.4809, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.6654447317123413, "rewards/margins": 1.8189523220062256, "rewards/rejected": -3.4843971729278564, "step": 6010 }, { "epoch": 0.8862063889297807, "grad_norm": 138.3334736509487, "learning_rate": 1.9468876365799054e-08, "logits/chosen": -1.1993780136108398, "logits/rejected": -0.815984845161438, "logps/chosen": -445.58349609375, "logps/rejected": -538.29443359375, "loss": 0.3615, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.7638969421386719, "rewards/margins": 1.5766842365264893, "rewards/rejected": -3.3405814170837402, "step": 6020 }, { "epoch": 0.8876784925658766, "grad_norm": 87.43403828640025, "learning_rate": 1.89748424203664e-08, "logits/chosen": -1.4031301736831665, "logits/rejected": -0.9359944462776184, "logps/chosen": -467.64886474609375, "logps/rejected": -519.8732299804688, "loss": 0.437, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.5854235887527466, "rewards/margins": 1.0862160921096802, "rewards/rejected": -2.6716394424438477, "step": 6030 }, { "epoch": 0.8891505962019726, "grad_norm": 75.77775039688505, "learning_rate": 1.8486910152764102e-08, "logits/chosen": -1.2027417421340942, "logits/rejected": -0.9280147552490234, "logps/chosen": -422.48388671875, "logps/rejected": -555.5831298828125, "loss": 0.4267, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.6530888080596924, "rewards/margins": 1.3602993488311768, "rewards/rejected": -3.013388156890869, "step": 6040 }, { "epoch": 0.8906226998380686, "grad_norm": 130.35340754132244, "learning_rate": 1.8005092449924292e-08, "logits/chosen": -1.3218846321105957, "logits/rejected": -1.231888771057129, "logps/chosen": -389.1402587890625, "logps/rejected": -465.5093688964844, "loss": 0.5728, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.763266921043396, "rewards/margins": 0.736245334148407, "rewards/rejected": -2.499511957168579, "step": 6050 }, { "epoch": 0.8920948034741646, "grad_norm": 190.32137629404687, "learning_rate": 1.7529402037285312e-08, "logits/chosen": -1.3747310638427734, "logits/rejected": -1.2134448289871216, "logps/chosen": -437.971435546875, "logps/rejected": -612.3836059570312, "loss": 0.4591, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.6321361064910889, "rewards/margins": 1.8617744445800781, "rewards/rejected": -3.493910551071167, "step": 6060 }, { "epoch": 0.8935669071102605, "grad_norm": 72.90205062854201, "learning_rate": 1.7059851478455594e-08, "logits/chosen": -1.336159348487854, "logits/rejected": -1.0463287830352783, "logps/chosen": -447.993896484375, "logps/rejected": -550.3836059570312, "loss": 0.4165, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.861944556236267, "rewards/margins": 1.318798303604126, "rewards/rejected": -3.1807429790496826, "step": 6070 }, { "epoch": 0.8950390107463565, "grad_norm": 98.9872237330582, "learning_rate": 1.6596453174882145e-08, "logits/chosen": -1.0017184019088745, "logits/rejected": -1.0421884059906006, "logps/chosen": -399.8570251464844, "logps/rejected": -596.8233642578125, "loss": 0.3481, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -2.0980629920959473, "rewards/margins": 1.2766414880752563, "rewards/rejected": -3.374704360961914, "step": 6080 }, { "epoch": 0.8965111143824526, "grad_norm": 59.48426601433193, "learning_rate": 1.6139219365522732e-08, "logits/chosen": -1.4107369184494019, "logits/rejected": -1.1122539043426514, "logps/chosen": -514.1741943359375, "logps/rejected": -623.8187866210938, "loss": 0.4501, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.7783997058868408, "rewards/margins": 1.2535008192062378, "rewards/rejected": -3.0319008827209473, "step": 6090 }, { "epoch": 0.8979832180185485, "grad_norm": 100.29092453904826, "learning_rate": 1.5688162126522853e-08, "logits/chosen": -1.4325015544891357, "logits/rejected": -0.9693012237548828, "logps/chosen": -461.904541015625, "logps/rejected": -509.4136657714844, "loss": 0.467, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.8348863124847412, "rewards/margins": 1.2343018054962158, "rewards/rejected": -3.069188356399536, "step": 6100 }, { "epoch": 0.8994553216546445, "grad_norm": 84.13420698485271, "learning_rate": 1.5243293370896554e-08, "logits/chosen": -1.4444586038589478, "logits/rejected": -1.0995537042617798, "logps/chosen": -440.20892333984375, "logps/rejected": -510.4673767089844, "loss": 0.4906, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -1.6723476648330688, "rewards/margins": 0.9577652215957642, "rewards/rejected": -2.630113124847412, "step": 6110 }, { "epoch": 0.9009274252907404, "grad_norm": 88.46133023122071, "learning_rate": 1.4804624848211933e-08, "logits/chosen": -1.1499179601669312, "logits/rejected": -0.7655227184295654, "logps/chosen": -420.9551696777344, "logps/rejected": -512.953125, "loss": 0.55, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.8270105123519897, "rewards/margins": 0.7387310266494751, "rewards/rejected": -2.5657413005828857, "step": 6120 }, { "epoch": 0.9023995289268365, "grad_norm": 112.99296743675342, "learning_rate": 1.4372168144280744e-08, "logits/chosen": -1.182217001914978, "logits/rejected": -0.7810528874397278, "logps/chosen": -432.20880126953125, "logps/rejected": -523.0076904296875, "loss": 0.5415, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.7092154026031494, "rewards/margins": 1.2686071395874023, "rewards/rejected": -2.9778225421905518, "step": 6130 }, { "epoch": 0.9038716325629325, "grad_norm": 121.96728716275769, "learning_rate": 1.3945934680852617e-08, "logits/chosen": -1.392909288406372, "logits/rejected": -0.8278254270553589, "logps/chosen": -475.64398193359375, "logps/rejected": -598.2657470703125, "loss": 0.4422, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.8078081607818604, "rewards/margins": 1.7844040393829346, "rewards/rejected": -3.592212200164795, "step": 6140 }, { "epoch": 0.9053437361990284, "grad_norm": 72.06405639321714, "learning_rate": 1.3525935715313074e-08, "logits/chosen": -1.1380715370178223, "logits/rejected": -0.898354172706604, "logps/chosen": -381.2165832519531, "logps/rejected": -534.9982299804688, "loss": 0.4306, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.9703505039215088, "rewards/margins": 1.1985515356063843, "rewards/rejected": -3.1689019203186035, "step": 6150 }, { "epoch": 0.9068158398351244, "grad_norm": 108.29273236635373, "learning_rate": 1.3112182340386412e-08, "logits/chosen": -1.0974479913711548, "logits/rejected": -0.8271905779838562, "logps/chosen": -378.52362060546875, "logps/rejected": -518.6552734375, "loss": 0.4617, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.7146676778793335, "rewards/margins": 1.60419499874115, "rewards/rejected": -3.3188629150390625, "step": 6160 }, { "epoch": 0.9082879434712203, "grad_norm": 101.71504234754592, "learning_rate": 1.2704685483842626e-08, "logits/chosen": -1.2984583377838135, "logits/rejected": -0.7792263627052307, "logps/chosen": -352.86566162109375, "logps/rejected": -436.63922119140625, "loss": 0.525, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.6000096797943115, "rewards/margins": 1.2597168684005737, "rewards/rejected": -2.859726667404175, "step": 6170 }, { "epoch": 0.9097600471073164, "grad_norm": 93.0214661593294, "learning_rate": 1.2303455908208915e-08, "logits/chosen": -1.4515659809112549, "logits/rejected": -1.148747205734253, "logps/chosen": -378.54266357421875, "logps/rejected": -454.61065673828125, "loss": 0.4494, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.527250051498413, "rewards/margins": 1.2347928285598755, "rewards/rejected": -2.76204252243042, "step": 6180 }, { "epoch": 0.9112321507434124, "grad_norm": 78.22321116231281, "learning_rate": 1.1908504210485337e-08, "logits/chosen": -1.262365460395813, "logits/rejected": -0.6896898150444031, "logps/chosen": -477.6587829589844, "logps/rejected": -525.4725341796875, "loss": 0.4674, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.6702487468719482, "rewards/margins": 1.1898547410964966, "rewards/rejected": -2.860103130340576, "step": 6190 }, { "epoch": 0.9127042543795083, "grad_norm": 68.9366431319535, "learning_rate": 1.1519840821864968e-08, "logits/chosen": -1.3926799297332764, "logits/rejected": -1.010058879852295, "logps/chosen": -483.1468200683594, "logps/rejected": -593.4761352539062, "loss": 0.4451, "rewards/accuracies": 0.7000000476837158, "rewards/chosen": -1.7724260091781616, "rewards/margins": 1.0586087703704834, "rewards/rejected": -2.8310346603393555, "step": 6200 }, { "epoch": 0.9141763580156043, "grad_norm": 255.98270815188275, "learning_rate": 1.113747600745829e-08, "logits/chosen": -1.3348357677459717, "logits/rejected": -1.1368389129638672, "logps/chosen": -411.146728515625, "logps/rejected": -558.9170532226562, "loss": 0.4938, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.572711706161499, "rewards/margins": 1.3852193355560303, "rewards/rejected": -2.9579310417175293, "step": 6210 }, { "epoch": 0.9156484616517003, "grad_norm": 171.3237826710108, "learning_rate": 1.0761419866022247e-08, "logits/chosen": -1.5173606872558594, "logits/rejected": -1.3542439937591553, "logps/chosen": -396.34625244140625, "logps/rejected": -472.72344970703125, "loss": 0.5572, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.4570844173431396, "rewards/margins": 1.0616931915283203, "rewards/rejected": -2.518777370452881, "step": 6220 }, { "epoch": 0.9171205652877963, "grad_norm": 96.76388140858792, "learning_rate": 1.039168232969348e-08, "logits/chosen": -1.2802958488464355, "logits/rejected": -1.0718050003051758, "logps/chosen": -438.5003967285156, "logps/rejected": -513.9151611328125, "loss": 0.5158, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.7826290130615234, "rewards/margins": 0.9541376233100891, "rewards/rejected": -2.736766815185547, "step": 6230 }, { "epoch": 0.9185926689238922, "grad_norm": 208.65795799594645, "learning_rate": 1.0028273163725898e-08, "logits/chosen": -1.3111271858215332, "logits/rejected": -1.194875955581665, "logps/chosen": -519.2174072265625, "logps/rejected": -596.43408203125, "loss": 0.4112, "rewards/accuracies": 0.76666659116745, "rewards/chosen": -1.882115364074707, "rewards/margins": 1.3816559314727783, "rewards/rejected": -3.2637715339660645, "step": 6240 }, { "epoch": 0.9200647725599882, "grad_norm": 110.28853386869616, "learning_rate": 9.671201966232889e-09, "logits/chosen": -1.2120047807693481, "logits/rejected": -1.106927514076233, "logps/chosen": -417.23101806640625, "logps/rejected": -551.600341796875, "loss": 0.4188, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.9461908340454102, "rewards/margins": 1.2960798740386963, "rewards/rejected": -3.2422707080841064, "step": 6250 }, { "epoch": 0.9215368761960842, "grad_norm": 65.0103326745241, "learning_rate": 9.320478167933659e-09, "logits/chosen": -1.3543187379837036, "logits/rejected": -1.2014297246932983, "logps/chosen": -451.106689453125, "logps/rejected": -591.7469482421875, "loss": 0.4751, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.7379400730133057, "rewards/margins": 1.395159125328064, "rewards/rejected": -3.13309907913208, "step": 6260 }, { "epoch": 0.9230089798321802, "grad_norm": 136.73147505961566, "learning_rate": 8.976111031904377e-09, "logits/chosen": -1.7549540996551514, "logits/rejected": -1.2522004842758179, "logps/chosen": -467.4527893066406, "logps/rejected": -527.7576904296875, "loss": 0.433, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -1.672555923461914, "rewards/margins": 1.3347749710083008, "rewards/rejected": -3.0073306560516357, "step": 6270 }, { "epoch": 0.9244810834682762, "grad_norm": 197.32138066123477, "learning_rate": 8.638109653333419e-09, "logits/chosen": -1.5915447473526, "logits/rejected": -1.028139591217041, "logps/chosen": -544.760498046875, "logps/rejected": -612.6495361328125, "loss": 0.4717, "rewards/accuracies": 0.8666666150093079, "rewards/chosen": -1.9581520557403564, "rewards/margins": 1.572818398475647, "rewards/rejected": -3.530970811843872, "step": 6280 }, { "epoch": 0.9259531871043721, "grad_norm": 88.9975140530782, "learning_rate": 8.306482959281103e-09, "logits/chosen": -1.1016359329223633, "logits/rejected": -0.9385136365890503, "logps/chosen": -415.51953125, "logps/rejected": -483.88916015625, "loss": 0.5574, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -2.1461730003356934, "rewards/margins": 0.9563619494438171, "rewards/rejected": -3.102534770965576, "step": 6290 }, { "epoch": 0.9274252907404681, "grad_norm": 136.91687710205173, "learning_rate": 7.981239708443971e-09, "logits/chosen": -1.3256946802139282, "logits/rejected": -0.9548309445381165, "logps/chosen": -456.28167724609375, "logps/rejected": -578.2469482421875, "loss": 0.5347, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.6343233585357666, "rewards/margins": 1.664646863937378, "rewards/rejected": -3.2989699840545654, "step": 6300 }, { "epoch": 0.9288973943765642, "grad_norm": 141.73391501968828, "learning_rate": 7.662388490923378e-09, "logits/chosen": -1.2113536596298218, "logits/rejected": -0.9762833714485168, "logps/chosen": -510.31536865234375, "logps/rejected": -675.57958984375, "loss": 0.4457, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -2.343094825744629, "rewards/margins": 1.4834887981414795, "rewards/rejected": -3.8265833854675293, "step": 6310 }, { "epoch": 0.9303694980126601, "grad_norm": 95.39226728017128, "learning_rate": 7.3499377279988e-09, "logits/chosen": -1.3638355731964111, "logits/rejected": -1.0438239574432373, "logps/chosen": -456.48736572265625, "logps/rejected": -506.72283935546875, "loss": 0.6138, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -2.035297155380249, "rewards/margins": 1.0015877485275269, "rewards/rejected": -3.0368847846984863, "step": 6320 }, { "epoch": 0.9318416016487561, "grad_norm": 83.18375545076209, "learning_rate": 7.043895671905248e-09, "logits/chosen": -1.3861024379730225, "logits/rejected": -1.381811499595642, "logps/chosen": -401.227783203125, "logps/rejected": -576.5643920898438, "loss": 0.4281, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.6423763036727905, "rewards/margins": 1.475299596786499, "rewards/rejected": -3.117676019668579, "step": 6330 }, { "epoch": 0.933313705284852, "grad_norm": 157.54591394824698, "learning_rate": 6.744270405615371e-09, "logits/chosen": -1.3402183055877686, "logits/rejected": -1.0885117053985596, "logps/chosen": -445.0760803222656, "logps/rejected": -595.1424560546875, "loss": 0.4372, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.563539743423462, "rewards/margins": 2.049320936203003, "rewards/rejected": -3.612860918045044, "step": 6340 }, { "epoch": 0.934785808920948, "grad_norm": 78.5332718700823, "learning_rate": 6.4510698426259165e-09, "logits/chosen": -1.3075374364852905, "logits/rejected": -1.0480823516845703, "logps/chosen": -421.25421142578125, "logps/rejected": -520.3898315429688, "loss": 0.3731, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.5807876586914062, "rewards/margins": 1.3279130458831787, "rewards/rejected": -2.908700704574585, "step": 6350 }, { "epoch": 0.936257912557044, "grad_norm": 110.06828598299056, "learning_rate": 6.164301726748838e-09, "logits/chosen": -1.358068823814392, "logits/rejected": -0.9059964418411255, "logps/chosen": -401.7532043457031, "logps/rejected": -521.3616333007812, "loss": 0.4533, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -1.3470642566680908, "rewards/margins": 1.7689765691757202, "rewards/rejected": -3.1160407066345215, "step": 6360 }, { "epoch": 0.93773001619314, "grad_norm": 153.47649978619683, "learning_rate": 5.883973631906736e-09, "logits/chosen": -1.3301823139190674, "logits/rejected": -1.1308788061141968, "logps/chosen": -468.0716857910156, "logps/rejected": -622.98291015625, "loss": 0.4996, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -2.2261788845062256, "rewards/margins": 1.54001784324646, "rewards/rejected": -3.7661972045898438, "step": 6370 }, { "epoch": 0.939202119829236, "grad_norm": 85.53426929713154, "learning_rate": 5.610092961932766e-09, "logits/chosen": -1.3597456216812134, "logits/rejected": -1.3059452772140503, "logps/chosen": -460.44781494140625, "logps/rejected": -603.2689208984375, "loss": 0.5525, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.913364052772522, "rewards/margins": 1.6412509679794312, "rewards/rejected": -3.554615020751953, "step": 6380 }, { "epoch": 0.9406742234653319, "grad_norm": 96.2904126112998, "learning_rate": 5.3426669503751355e-09, "logits/chosen": -1.5512850284576416, "logits/rejected": -0.9863389730453491, "logps/chosen": -525.7786254882812, "logps/rejected": -536.9043579101562, "loss": 0.4689, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -2.06516695022583, "rewards/margins": 0.9437375068664551, "rewards/rejected": -3.008904457092285, "step": 6390 }, { "epoch": 0.942146327101428, "grad_norm": 142.36380621173555, "learning_rate": 5.081702660306025e-09, "logits/chosen": -1.478801965713501, "logits/rejected": -0.8947819471359253, "logps/chosen": -473.7356872558594, "logps/rejected": -640.308837890625, "loss": 0.5642, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -2.126918315887451, "rewards/margins": 1.6010997295379639, "rewards/rejected": -3.728017807006836, "step": 6400 }, { "epoch": 0.9436184307375239, "grad_norm": 152.4371615430786, "learning_rate": 4.827206984135024e-09, "logits/chosen": -1.2242169380187988, "logits/rejected": -1.00106942653656, "logps/chosen": -406.49755859375, "logps/rejected": -521.4188842773438, "loss": 0.5218, "rewards/accuracies": 0.76666659116745, "rewards/chosen": -2.009997844696045, "rewards/margins": 1.1500599384307861, "rewards/rejected": -3.160057544708252, "step": 6410 }, { "epoch": 0.9450905343736199, "grad_norm": 109.5170505315079, "learning_rate": 4.579186643427241e-09, "logits/chosen": -1.4265880584716797, "logits/rejected": -1.1079994440078735, "logps/chosen": -517.7324829101562, "logps/rejected": -550.4271240234375, "loss": 0.5119, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.8106954097747803, "rewards/margins": 1.1361420154571533, "rewards/rejected": -2.9468376636505127, "step": 6420 }, { "epoch": 0.9465626380097159, "grad_norm": 120.06199176900785, "learning_rate": 4.337648188725618e-09, "logits/chosen": -1.2486460208892822, "logits/rejected": -0.9750927686691284, "logps/chosen": -422.97454833984375, "logps/rejected": -530.0245971679688, "loss": 0.5378, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.3605560064315796, "rewards/margins": 1.57229745388031, "rewards/rejected": -2.9328534603118896, "step": 6430 }, { "epoch": 0.9480347416458119, "grad_norm": 121.72026440077309, "learning_rate": 4.102597999377927e-09, "logits/chosen": -1.3509511947631836, "logits/rejected": -1.1400359869003296, "logps/chosen": -549.1176147460938, "logps/rejected": -692.47509765625, "loss": 0.4896, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -2.322995662689209, "rewards/margins": 1.242288589477539, "rewards/rejected": -3.565284013748169, "step": 6440 }, { "epoch": 0.9495068452819079, "grad_norm": 221.28721364142348, "learning_rate": 3.874042283368406e-09, "logits/chosen": -1.3209785223007202, "logits/rejected": -1.0659959316253662, "logps/chosen": -450.44146728515625, "logps/rejected": -583.3470458984375, "loss": 0.4932, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -2.1836166381835938, "rewards/margins": 1.1096569299697876, "rewards/rejected": -3.2932732105255127, "step": 6450 }, { "epoch": 0.9509789489180038, "grad_norm": 100.69938687648991, "learning_rate": 3.6519870771536952e-09, "logits/chosen": -1.286990761756897, "logits/rejected": -1.3209233283996582, "logps/chosen": -437.3722229003906, "logps/rejected": -582.5029296875, "loss": 0.4342, "rewards/accuracies": 0.6999999284744263, "rewards/chosen": -1.9360355138778687, "rewards/margins": 0.7741468548774719, "rewards/rejected": -2.7101826667785645, "step": 6460 }, { "epoch": 0.9524510525540998, "grad_norm": 113.24796094369674, "learning_rate": 3.436438245503409e-09, "logits/chosen": -1.1490650177001953, "logits/rejected": -1.061572790145874, "logps/chosen": -461.55877685546875, "logps/rejected": -567.5509033203125, "loss": 0.558, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.6923081874847412, "rewards/margins": 1.226409673690796, "rewards/rejected": -2.918717861175537, "step": 6470 }, { "epoch": 0.9539231561901957, "grad_norm": 79.79596970660135, "learning_rate": 3.2274014813452875e-09, "logits/chosen": -1.1326738595962524, "logits/rejected": -0.8211013078689575, "logps/chosen": -428.6504821777344, "logps/rejected": -677.7749633789062, "loss": 0.4101, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -2.111638307571411, "rewards/margins": 1.9096629619598389, "rewards/rejected": -4.021300792694092, "step": 6480 }, { "epoch": 0.9553952598262918, "grad_norm": 125.84258421914738, "learning_rate": 3.0248823056148175e-09, "logits/chosen": -1.1957733631134033, "logits/rejected": -1.4037882089614868, "logps/chosen": -433.4906311035156, "logps/rejected": -587.3173217773438, "loss": 0.4584, "rewards/accuracies": 0.73333340883255, "rewards/chosen": -1.8290573358535767, "rewards/margins": 1.4790170192718506, "rewards/rejected": -3.3080742359161377, "step": 6490 }, { "epoch": 0.9568673634623878, "grad_norm": 150.38562504197765, "learning_rate": 2.828886067109404e-09, "logits/chosen": -1.4119644165039062, "logits/rejected": -1.2277817726135254, "logps/chosen": -508.92205810546875, "logps/rejected": -642.3756103515625, "loss": 0.4982, "rewards/accuracies": 0.8333333134651184, "rewards/chosen": -2.104766368865967, "rewards/margins": 1.6219040155410767, "rewards/rejected": -3.7266697883605957, "step": 6500 }, { "epoch": 0.9583394670984837, "grad_norm": 98.17333195909225, "learning_rate": 2.639417942347122e-09, "logits/chosen": -1.3974770307540894, "logits/rejected": -1.0616443157196045, "logps/chosen": -366.06060791015625, "logps/rejected": -589.9656982421875, "loss": 0.5112, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.51221764087677, "rewards/margins": 2.0129287242889404, "rewards/rejected": -3.525146007537842, "step": 6510 }, { "epoch": 0.9598115707345797, "grad_norm": 187.6051838583204, "learning_rate": 2.456482935429993e-09, "logits/chosen": -1.3879002332687378, "logits/rejected": -1.0190473794937134, "logps/chosen": -480.7325744628906, "logps/rejected": -579.7218017578125, "loss": 0.6549, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -2.0841310024261475, "rewards/margins": 1.207476258277893, "rewards/rejected": -3.291607618331909, "step": 6520 }, { "epoch": 0.9612836743706757, "grad_norm": 87.85131668882238, "learning_rate": 2.2800858779118417e-09, "logits/chosen": -1.238096833229065, "logits/rejected": -1.061269760131836, "logps/chosen": -429.9205017089844, "logps/rejected": -571.918212890625, "loss": 0.6518, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.7396520376205444, "rewards/margins": 1.5972769260406494, "rewards/rejected": -3.336928606033325, "step": 6530 }, { "epoch": 0.9627557780067717, "grad_norm": 122.8780597463822, "learning_rate": 2.1102314286705334e-09, "logits/chosen": -1.3741991519927979, "logits/rejected": -1.2921290397644043, "logps/chosen": -510.30975341796875, "logps/rejected": -513.4083251953125, "loss": 0.486, "rewards/accuracies": 0.5666666626930237, "rewards/chosen": -1.8251821994781494, "rewards/margins": 0.5057021379470825, "rewards/rejected": -2.3308842182159424, "step": 6540 }, { "epoch": 0.9642278816428677, "grad_norm": 91.14450887378497, "learning_rate": 1.9469240737852445e-09, "logits/chosen": -1.2866744995117188, "logits/rejected": -1.1259269714355469, "logps/chosen": -322.0516357421875, "logps/rejected": -511.1654357910156, "loss": 0.4444, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.310871958732605, "rewards/margins": 1.750274419784546, "rewards/rejected": -3.0611460208892822, "step": 6550 }, { "epoch": 0.9656999852789636, "grad_norm": 89.35137950308767, "learning_rate": 1.790168126417635e-09, "logits/chosen": -1.3406840562820435, "logits/rejected": -0.8247464895248413, "logps/chosen": -352.471923828125, "logps/rejected": -486.70989990234375, "loss": 0.6235, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.3377752304077148, "rewards/margins": 1.5931618213653564, "rewards/rejected": -2.9309370517730713, "step": 6560 }, { "epoch": 0.9671720889150596, "grad_norm": 91.35928129358597, "learning_rate": 1.6399677266982214e-09, "logits/chosen": -1.3589528799057007, "logits/rejected": -1.1516953706741333, "logps/chosen": -499.2710876464844, "logps/rejected": -571.6011962890625, "loss": 0.4366, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.0111379623413086, "rewards/margins": 1.0724924802780151, "rewards/rejected": -3.0836308002471924, "step": 6570 }, { "epoch": 0.9686441925511556, "grad_norm": 243.42171472032464, "learning_rate": 1.4963268416167685e-09, "logits/chosen": -1.2164876461029053, "logits/rejected": -0.8679919242858887, "logps/chosen": -421.5487365722656, "logps/rejected": -547.1917724609375, "loss": 0.5454, "rewards/accuracies": 0.7333333492279053, "rewards/chosen": -1.509100079536438, "rewards/margins": 1.6187273263931274, "rewards/rejected": -3.1278271675109863, "step": 6580 }, { "epoch": 0.9701162961872516, "grad_norm": 90.16390237464518, "learning_rate": 1.3592492649177056e-09, "logits/chosen": -1.0717471837997437, "logits/rejected": -0.9164643287658691, "logps/chosen": -423.8712463378906, "logps/rejected": -534.1849365234375, "loss": 0.4693, "rewards/accuracies": 0.8666666746139526, "rewards/chosen": -1.6814769506454468, "rewards/margins": 1.592142105102539, "rewards/rejected": -3.2736194133758545, "step": 6590 }, { "epoch": 0.9715883998233475, "grad_norm": 167.80484523337927, "learning_rate": 1.2287386169998747e-09, "logits/chosen": -1.6180499792099, "logits/rejected": -1.0677508115768433, "logps/chosen": -430.90753173828125, "logps/rejected": -562.3106079101562, "loss": 0.4464, "rewards/accuracies": 0.7999999523162842, "rewards/chosen": -1.796453833580017, "rewards/margins": 1.643140435218811, "rewards/rejected": -3.439594268798828, "step": 6600 }, { "epoch": 0.9730605034594435, "grad_norm": 121.2607288611665, "learning_rate": 1.1047983448209397e-09, "logits/chosen": -1.3967403173446655, "logits/rejected": -1.1385594606399536, "logps/chosen": -437.872802734375, "logps/rejected": -550.6277465820312, "loss": 0.4394, "rewards/accuracies": 0.9000000953674316, "rewards/chosen": -1.5407131910324097, "rewards/margins": 1.4084869623184204, "rewards/rejected": -2.94920015335083, "step": 6610 }, { "epoch": 0.9745326070955396, "grad_norm": 62.21049188639438, "learning_rate": 9.87431721806209e-10, "logits/chosen": -1.5890763998031616, "logits/rejected": -1.2994145154953003, "logps/chosen": -421.8890686035156, "logps/rejected": -507.55224609375, "loss": 0.4199, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.5588992834091187, "rewards/margins": 1.3105344772338867, "rewards/rejected": -2.869433879852295, "step": 6620 }, { "epoch": 0.9760047107316355, "grad_norm": 63.232338001046614, "learning_rate": 8.766418477623716e-10, "logits/chosen": -1.2572543621063232, "logits/rejected": -0.9364638328552246, "logps/chosen": -413.59002685546875, "logps/rejected": -499.2174377441406, "loss": 0.3867, "rewards/accuracies": 0.7666667103767395, "rewards/chosen": -1.4785820245742798, "rewards/margins": 1.3318545818328857, "rewards/rejected": -2.810436725616455, "step": 6630 }, { "epoch": 0.9774768143677315, "grad_norm": 157.68941789773905, "learning_rate": 7.724316487954796e-10, "logits/chosen": -1.3735524415969849, "logits/rejected": -1.0742418766021729, "logps/chosen": -404.0660400390625, "logps/rejected": -567.3262939453125, "loss": 0.5536, "rewards/accuracies": 0.76666659116745, "rewards/chosen": -1.6344239711761475, "rewards/margins": 1.5732135772705078, "rewards/rejected": -3.207637310028076, "step": 6640 }, { "epoch": 0.9789489180038274, "grad_norm": 127.63106630389774, "learning_rate": 6.748038772337595e-10, "logits/chosen": -1.3165947198867798, "logits/rejected": -0.9859461784362793, "logps/chosen": -435.93133544921875, "logps/rejected": -533.8653564453125, "loss": 0.5505, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -1.759289026260376, "rewards/margins": 1.2258495092391968, "rewards/rejected": -2.985138416290283, "step": 6650 }, { "epoch": 0.9804210216399234, "grad_norm": 176.17770544447473, "learning_rate": 5.837611115549201e-10, "logits/chosen": -1.5459668636322021, "logits/rejected": -0.888058066368103, "logps/chosen": -464.20172119140625, "logps/rejected": -534.7344970703125, "loss": 0.5347, "rewards/accuracies": 0.7333332896232605, "rewards/chosen": -1.4714539051055908, "rewards/margins": 1.4597190618515015, "rewards/rejected": -2.9311728477478027, "step": 6660 }, { "epoch": 0.9818931252760195, "grad_norm": 166.41130513975992, "learning_rate": 4.993057563179303e-10, "logits/chosen": -1.5793631076812744, "logits/rejected": -1.1331535577774048, "logps/chosen": -499.2445373535156, "logps/rejected": -556.5195922851562, "loss": 0.3825, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.7106945514678955, "rewards/margins": 1.5195393562316895, "rewards/rejected": -3.230233669281006, "step": 6670 }, { "epoch": 0.9833652289121154, "grad_norm": 236.0814983966395, "learning_rate": 4.2144004209962445e-10, "logits/chosen": -1.285900592803955, "logits/rejected": -1.2104876041412354, "logps/chosen": -454.2847595214844, "logps/rejected": -510.22247314453125, "loss": 0.4775, "rewards/accuracies": 0.6333333253860474, "rewards/chosen": -1.477925419807434, "rewards/margins": 0.8418137431144714, "rewards/rejected": -2.3197388648986816, "step": 6680 }, { "epoch": 0.9848373325482114, "grad_norm": 68.83875850322735, "learning_rate": 3.501660254358607e-10, "logits/chosen": -1.2782337665557861, "logits/rejected": -0.8187475204467773, "logps/chosen": -432.896728515625, "logps/rejected": -539.8255615234375, "loss": 0.4564, "rewards/accuracies": 0.6666666865348816, "rewards/chosen": -2.147003650665283, "rewards/margins": 1.3009148836135864, "rewards/rejected": -3.447918653488159, "step": 6690 }, { "epoch": 0.9863094361843073, "grad_norm": 67.97220529141755, "learning_rate": 2.854855887669538e-10, "logits/chosen": -1.3216626644134521, "logits/rejected": -1.440431833267212, "logps/chosen": -487.4225158691406, "logps/rejected": -620.2841796875, "loss": 0.4624, "rewards/accuracies": 0.73333340883255, "rewards/chosen": -1.8056331872940063, "rewards/margins": 1.089308738708496, "rewards/rejected": -2.894942045211792, "step": 6700 }, { "epoch": 0.9877815398204034, "grad_norm": 121.32963259885697, "learning_rate": 2.274004403882146e-10, "logits/chosen": -1.367511510848999, "logits/rejected": -1.0503766536712646, "logps/chosen": -444.0887756347656, "logps/rejected": -604.0455322265625, "loss": 0.4112, "rewards/accuracies": 0.76666659116745, "rewards/chosen": -1.7945635318756104, "rewards/margins": 1.5105736255645752, "rewards/rejected": -3.3051364421844482, "step": 6710 }, { "epoch": 0.9892536434564994, "grad_norm": 142.330296107857, "learning_rate": 1.7591211440468047e-10, "logits/chosen": -1.2823817729949951, "logits/rejected": -0.941415011882782, "logps/chosen": -417.048095703125, "logps/rejected": -582.2371826171875, "loss": 0.5327, "rewards/accuracies": 0.8333333730697632, "rewards/chosen": -1.5511300563812256, "rewards/margins": 1.97564697265625, "rewards/rejected": -3.5267767906188965, "step": 6720 }, { "epoch": 0.9907257470925953, "grad_norm": 177.86458994718856, "learning_rate": 1.3102197069067567e-10, "logits/chosen": -1.2806065082550049, "logits/rejected": -0.7323794960975647, "logps/chosen": -455.69659423828125, "logps/rejected": -587.4266357421875, "loss": 0.4243, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -1.9524253606796265, "rewards/margins": 1.4952733516693115, "rewards/rejected": -3.4476981163024902, "step": 6730 }, { "epoch": 0.9921978507286913, "grad_norm": 103.03628687622312, "learning_rate": 9.273119485386783e-11, "logits/chosen": -1.3981177806854248, "logits/rejected": -1.2410085201263428, "logps/chosen": -490.24981689453125, "logps/rejected": -645.2311401367188, "loss": 0.5337, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -2.1277194023132324, "rewards/margins": 1.3783842325210571, "rewards/rejected": -3.506103038787842, "step": 6740 }, { "epoch": 0.9936699543647873, "grad_norm": 116.5064689963164, "learning_rate": 6.104079820390407e-11, "logits/chosen": -1.4266693592071533, "logits/rejected": -1.2530936002731323, "logps/chosen": -511.64459228515625, "logps/rejected": -556.6649169921875, "loss": 0.5815, "rewards/accuracies": 0.5666666626930237, "rewards/chosen": -2.1102561950683594, "rewards/margins": 0.519591212272644, "rewards/rejected": -2.629847288131714, "step": 6750 }, { "epoch": 0.9951420580008833, "grad_norm": 97.30029923815557, "learning_rate": 3.595161772582123e-11, "logits/chosen": -1.3649976253509521, "logits/rejected": -1.0474755764007568, "logps/chosen": -344.64178466796875, "logps/rejected": -458.5502014160156, "loss": 0.522, "rewards/accuracies": 0.7666666507720947, "rewards/chosen": -1.2290351390838623, "rewards/margins": 1.540971279144287, "rewards/rejected": -2.7700061798095703, "step": 6760 }, { "epoch": 0.9966141616369792, "grad_norm": 179.81145239599982, "learning_rate": 1.7464316057785866e-11, "logits/chosen": -1.4237736463546753, "logits/rejected": -1.0620884895324707, "logps/chosen": -453.30889892578125, "logps/rejected": -555.2007446289062, "loss": 0.4954, "rewards/accuracies": 0.699999988079071, "rewards/chosen": -1.6915595531463623, "rewards/margins": 1.1138288974761963, "rewards/rejected": -2.8053882122039795, "step": 6770 }, { "epoch": 0.9980862652730752, "grad_norm": 193.13835305388184, "learning_rate": 5.5793814737192805e-12, "logits/chosen": -1.2985987663269043, "logits/rejected": -1.2492955923080444, "logps/chosen": -500.07122802734375, "logps/rejected": -522.789306640625, "loss": 0.5305, "rewards/accuracies": 0.6333333849906921, "rewards/chosen": -2.213953971862793, "rewards/margins": 0.6303444504737854, "rewards/rejected": -2.8442983627319336, "step": 6780 }, { "epoch": 0.9995583689091712, "grad_norm": 116.06156637115119, "learning_rate": 2.9712787039115617e-13, "logits/chosen": -1.0107321739196777, "logits/rejected": -0.636509120464325, "logps/chosen": -403.2305603027344, "logps/rejected": -516.6555786132812, "loss": 0.5284, "rewards/accuracies": 0.6666666269302368, "rewards/chosen": -1.8709781169891357, "rewards/margins": 1.2671401500701904, "rewards/rejected": -3.138118267059326, "step": 6790 }, { "epoch": 1.0, "step": 6793, "total_flos": 0.0, "train_loss": 0.44209186683851615, "train_runtime": 28094.4071, "train_samples_per_second": 2.176, "train_steps_per_second": 0.242 } ], "logging_steps": 10, "max_steps": 6793, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 3, "trial_name": null, "trial_params": null }