diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,4522 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 100, + "global_step": 2889, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 1.7301038062283737e-09, + "logits/chosen": -2.8762407302856445, + "logits/rejected": -3.173783540725708, + "logps/chosen": -321.2347412109375, + "logps/rejected": -216.31759643554688, + "loss": 0.6931, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "epoch": 0.01, + "learning_rate": 1.7301038062283738e-08, + "logits/chosen": -2.934302568435669, + "logits/rejected": -2.866340398788452, + "logps/chosen": -282.9244384765625, + "logps/rejected": -240.00062561035156, + "loss": 0.6946, + "rewards/accuracies": 0.4305555522441864, + "rewards/chosen": -0.00089820078574121, + "rewards/margins": 0.005787495523691177, + "rewards/rejected": -0.006685695610940456, + "step": 10 + }, + { + "epoch": 0.02, + "learning_rate": 3.4602076124567476e-08, + "logits/chosen": -3.0233330726623535, + "logits/rejected": -2.6111900806427, + "logps/chosen": -242.19534301757812, + "logps/rejected": -218.64111328125, + "loss": 0.6932, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": 0.004412095528095961, + "rewards/margins": 0.008006840944290161, + "rewards/rejected": -0.003594745649024844, + "step": 20 + }, + { + "epoch": 0.03, + "learning_rate": 5.190311418685121e-08, + "logits/chosen": -2.994711399078369, + "logits/rejected": -2.569889545440674, + "logps/chosen": -289.1451110839844, + "logps/rejected": -205.26162719726562, + "loss": 0.6903, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": 0.01539570651948452, + "rewards/margins": 0.015161572024226189, + "rewards/rejected": 0.00023413417511619627, + "step": 30 + }, + { + "epoch": 0.04, + "learning_rate": 6.920415224913495e-08, + "logits/chosen": -2.974442958831787, + "logits/rejected": -2.7724432945251465, + "logps/chosen": -243.93896484375, + "logps/rejected": -226.4752655029297, + "loss": 0.6819, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.0148529764264822, + "rewards/margins": 0.015147365629673004, + "rewards/rejected": -0.0002943886793218553, + "step": 40 + }, + { + "epoch": 0.05, + "learning_rate": 8.650519031141867e-08, + "logits/chosen": -2.989952802658081, + "logits/rejected": -2.679442882537842, + "logps/chosen": -256.74560546875, + "logps/rejected": -226.18637084960938, + "loss": 0.6649, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.023135431110858917, + "rewards/margins": 0.03898553177714348, + "rewards/rejected": -0.01585010625422001, + "step": 50 + }, + { + "epoch": 0.06, + "learning_rate": 1.0380622837370242e-07, + "logits/chosen": -3.1331353187561035, + "logits/rejected": -2.938162326812744, + "logps/chosen": -353.51629638671875, + "logps/rejected": -251.57357788085938, + "loss": 0.6388, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.09346623718738556, + "rewards/margins": 0.13534647226333618, + "rewards/rejected": -0.041880227625370026, + "step": 60 + }, + { + "epoch": 0.07, + "learning_rate": 1.2110726643598616e-07, + "logits/chosen": -3.092569351196289, + "logits/rejected": -2.9506618976593018, + "logps/chosen": -269.38006591796875, + "logps/rejected": -253.18295288085938, + "loss": 0.6073, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 0.12881746888160706, + "rewards/margins": 0.19464503228664398, + "rewards/rejected": -0.06582757830619812, + "step": 70 + }, + { + "epoch": 0.08, + "learning_rate": 1.384083044982699e-07, + "logits/chosen": -3.1748270988464355, + "logits/rejected": -2.881019115447998, + "logps/chosen": -318.3662109375, + "logps/rejected": -266.37249755859375, + "loss": 0.5724, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.20069709420204163, + "rewards/margins": 0.47188153862953186, + "rewards/rejected": -0.2711844742298126, + "step": 80 + }, + { + "epoch": 0.09, + "learning_rate": 1.5570934256055362e-07, + "logits/chosen": -2.9505438804626465, + "logits/rejected": -2.949632167816162, + "logps/chosen": -280.816650390625, + "logps/rejected": -239.5382843017578, + "loss": 0.5404, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.19082725048065186, + "rewards/margins": 0.41627758741378784, + "rewards/rejected": -0.22545035183429718, + "step": 90 + }, + { + "epoch": 0.1, + "learning_rate": 1.7301038062283734e-07, + "logits/chosen": -2.823728084564209, + "logits/rejected": -3.019169569015503, + "logps/chosen": -284.28936767578125, + "logps/rejected": -280.98675537109375, + "loss": 0.5051, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.16983866691589355, + "rewards/margins": 0.6204549670219421, + "rewards/rejected": -0.45061635971069336, + "step": 100 + }, + { + "epoch": 0.1, + "eval_logits/chosen": -3.043107271194458, + "eval_logits/rejected": -2.7438313961029053, + "eval_logps/chosen": -297.5411682128906, + "eval_logps/rejected": -246.62863159179688, + "eval_loss": 0.5180009603500366, + "eval_rewards/accuracies": 0.7182539701461792, + "eval_rewards/chosen": 0.14754721522331238, + "eval_rewards/margins": 0.5429435968399048, + "eval_rewards/rejected": -0.3953963816165924, + "eval_runtime": 130.1164, + "eval_samples_per_second": 15.371, + "eval_steps_per_second": 0.484, + "step": 100 + }, + { + "epoch": 0.11, + "learning_rate": 1.903114186851211e-07, + "logits/chosen": -3.098701238632202, + "logits/rejected": -2.781334638595581, + "logps/chosen": -300.07354736328125, + "logps/rejected": -215.4860382080078, + "loss": 0.4786, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": 0.17965184152126312, + "rewards/margins": 0.6233574748039246, + "rewards/rejected": -0.44370561838150024, + "step": 110 + }, + { + "epoch": 0.12, + "learning_rate": 2.0761245674740483e-07, + "logits/chosen": -3.1609914302825928, + "logits/rejected": -2.6632208824157715, + "logps/chosen": -319.6437072753906, + "logps/rejected": -274.22991943359375, + "loss": 0.4765, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": 0.04884842038154602, + "rewards/margins": 0.6324900388717651, + "rewards/rejected": -0.5836416482925415, + "step": 120 + }, + { + "epoch": 0.13, + "learning_rate": 2.2491349480968858e-07, + "logits/chosen": -3.015540599822998, + "logits/rejected": -3.109987497329712, + "logps/chosen": -315.15313720703125, + "logps/rejected": -275.33453369140625, + "loss": 0.4788, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 0.16350099444389343, + "rewards/margins": 1.0702369213104248, + "rewards/rejected": -0.906735897064209, + "step": 130 + }, + { + "epoch": 0.15, + "learning_rate": 2.422145328719723e-07, + "logits/chosen": -3.0585391521453857, + "logits/rejected": -2.459653377532959, + "logps/chosen": -303.3382873535156, + "logps/rejected": -253.593017578125, + "loss": 0.4977, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.09562154114246368, + "rewards/margins": 0.9685525894165039, + "rewards/rejected": -1.0641741752624512, + "step": 140 + }, + { + "epoch": 0.16, + "learning_rate": 2.59515570934256e-07, + "logits/chosen": -3.066427230834961, + "logits/rejected": -2.8052279949188232, + "logps/chosen": -297.28729248046875, + "logps/rejected": -237.7314910888672, + "loss": 0.4381, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.19543597102165222, + "rewards/margins": 1.5181944370269775, + "rewards/rejected": -1.322758436203003, + "step": 150 + }, + { + "epoch": 0.17, + "learning_rate": 2.768166089965398e-07, + "logits/chosen": -3.1258957386016846, + "logits/rejected": -2.8570971488952637, + "logps/chosen": -326.80047607421875, + "logps/rejected": -266.1025695800781, + "loss": 0.4518, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 0.3090304136276245, + "rewards/margins": 1.3269016742706299, + "rewards/rejected": -1.017871379852295, + "step": 160 + }, + { + "epoch": 0.18, + "learning_rate": 2.941176470588235e-07, + "logits/chosen": -3.1422178745269775, + "logits/rejected": -2.748126268386841, + "logps/chosen": -304.7181091308594, + "logps/rejected": -248.3983612060547, + "loss": 0.4191, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 0.31797105073928833, + "rewards/margins": 1.5963741540908813, + "rewards/rejected": -1.2784031629562378, + "step": 170 + }, + { + "epoch": 0.19, + "learning_rate": 3.1141868512110725e-07, + "logits/chosen": -3.1268668174743652, + "logits/rejected": -2.895526885986328, + "logps/chosen": -293.18341064453125, + "logps/rejected": -238.168701171875, + "loss": 0.4801, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 0.055577002465724945, + "rewards/margins": 1.1816781759262085, + "rewards/rejected": -1.1261012554168701, + "step": 180 + }, + { + "epoch": 0.2, + "learning_rate": 3.28719723183391e-07, + "logits/chosen": -3.063249111175537, + "logits/rejected": -2.6634368896484375, + "logps/chosen": -279.2200927734375, + "logps/rejected": -234.94046020507812, + "loss": 0.4367, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.3853099048137665, + "rewards/margins": 1.4658257961273193, + "rewards/rejected": -1.0805158615112305, + "step": 190 + }, + { + "epoch": 0.21, + "learning_rate": 3.460207612456747e-07, + "logits/chosen": -3.158604621887207, + "logits/rejected": -2.772695541381836, + "logps/chosen": -297.8023681640625, + "logps/rejected": -254.5688934326172, + "loss": 0.4321, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.2604522109031677, + "rewards/margins": 1.288292646408081, + "rewards/rejected": -1.027840495109558, + "step": 200 + }, + { + "epoch": 0.21, + "eval_logits/chosen": -3.0543432235717773, + "eval_logits/rejected": -2.7577624320983887, + "eval_logps/chosen": -297.6632080078125, + "eval_logps/rejected": -252.20359802246094, + "eval_loss": 0.4374702274799347, + "eval_rewards/accuracies": 0.7539682388305664, + "eval_rewards/chosen": 0.13534867763519287, + "eval_rewards/margins": 1.0882411003112793, + "eval_rewards/rejected": -0.9528924822807312, + "eval_runtime": 129.9621, + "eval_samples_per_second": 15.389, + "eval_steps_per_second": 0.485, + "step": 200 + }, + { + "epoch": 0.22, + "learning_rate": 3.633217993079585e-07, + "logits/chosen": -3.1798148155212402, + "logits/rejected": -2.5849387645721436, + "logps/chosen": -361.23822021484375, + "logps/rejected": -268.04400634765625, + "loss": 0.3972, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 0.4466512203216553, + "rewards/margins": 1.7230627536773682, + "rewards/rejected": -1.2764114141464233, + "step": 210 + }, + { + "epoch": 0.23, + "learning_rate": 3.806228373702422e-07, + "logits/chosen": -3.0083553791046143, + "logits/rejected": -2.7477853298187256, + "logps/chosen": -281.7891540527344, + "logps/rejected": -229.61257934570312, + "loss": 0.41, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.09162549674510956, + "rewards/margins": 1.5220754146575928, + "rewards/rejected": -1.6137008666992188, + "step": 220 + }, + { + "epoch": 0.24, + "learning_rate": 3.979238754325259e-07, + "logits/chosen": -3.013458490371704, + "logits/rejected": -3.002962827682495, + "logps/chosen": -330.75360107421875, + "logps/rejected": -287.80621337890625, + "loss": 0.4135, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.23488721251487732, + "rewards/margins": 1.8438904285430908, + "rewards/rejected": -1.6090030670166016, + "step": 230 + }, + { + "epoch": 0.25, + "learning_rate": 4.1522491349480966e-07, + "logits/chosen": -3.1606154441833496, + "logits/rejected": -2.604207754135132, + "logps/chosen": -273.66851806640625, + "logps/rejected": -227.6764373779297, + "loss": 0.4219, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.22179162502288818, + "rewards/margins": 1.2629636526107788, + "rewards/rejected": -1.484755516052246, + "step": 240 + }, + { + "epoch": 0.26, + "learning_rate": 4.325259515570934e-07, + "logits/chosen": -3.045926570892334, + "logits/rejected": -2.9288675785064697, + "logps/chosen": -269.4823913574219, + "logps/rejected": -231.5515594482422, + "loss": 0.4175, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1447521448135376, + "rewards/margins": 1.5314770936965942, + "rewards/rejected": -1.6762291193008423, + "step": 250 + }, + { + "epoch": 0.27, + "learning_rate": 4.4982698961937715e-07, + "logits/chosen": -2.953214168548584, + "logits/rejected": -2.881971597671509, + "logps/chosen": -279.1170349121094, + "logps/rejected": -259.5255126953125, + "loss": 0.4477, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.6336969137191772, + "rewards/margins": 1.3820281028747559, + "rewards/rejected": -2.0157248973846436, + "step": 260 + }, + { + "epoch": 0.28, + "learning_rate": 4.671280276816609e-07, + "logits/chosen": -2.955901861190796, + "logits/rejected": -2.9216983318328857, + "logps/chosen": -306.7757263183594, + "logps/rejected": -248.06576538085938, + "loss": 0.4471, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -0.4410484731197357, + "rewards/margins": 1.6437104940414429, + "rewards/rejected": -2.08475923538208, + "step": 270 + }, + { + "epoch": 0.29, + "learning_rate": 4.844290657439446e-07, + "logits/chosen": -3.0967202186584473, + "logits/rejected": -2.701328992843628, + "logps/chosen": -343.8622741699219, + "logps/rejected": -255.25466918945312, + "loss": 0.4219, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.018992478027939796, + "rewards/margins": 1.7166669368743896, + "rewards/rejected": -1.7356598377227783, + "step": 280 + }, + { + "epoch": 0.3, + "learning_rate": 4.998076923076923e-07, + "logits/chosen": -3.1110892295837402, + "logits/rejected": -2.551694869995117, + "logps/chosen": -322.7112121582031, + "logps/rejected": -257.21002197265625, + "loss": 0.3876, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 0.04921429231762886, + "rewards/margins": 1.8386141061782837, + "rewards/rejected": -1.7893998622894287, + "step": 290 + }, + { + "epoch": 0.31, + "learning_rate": 4.978846153846153e-07, + "logits/chosen": -3.186720371246338, + "logits/rejected": -2.7246062755584717, + "logps/chosen": -291.0187072753906, + "logps/rejected": -258.8619384765625, + "loss": 0.3848, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 0.12159641087055206, + "rewards/margins": 2.314002275466919, + "rewards/rejected": -2.1924057006835938, + "step": 300 + }, + { + "epoch": 0.31, + "eval_logits/chosen": -3.05080509185791, + "eval_logits/rejected": -2.7592227458953857, + "eval_logps/chosen": -303.83013916015625, + "eval_logps/rejected": -261.59564208984375, + "eval_loss": 0.43010157346725464, + "eval_rewards/accuracies": 0.7301587462425232, + "eval_rewards/chosen": -0.4813474714756012, + "eval_rewards/margins": 1.4107486009597778, + "eval_rewards/rejected": -1.8920958042144775, + "eval_runtime": 129.8338, + "eval_samples_per_second": 15.404, + "eval_steps_per_second": 0.485, + "step": 300 + }, + { + "epoch": 0.32, + "learning_rate": 4.959615384615384e-07, + "logits/chosen": -3.025524616241455, + "logits/rejected": -2.8964145183563232, + "logps/chosen": -284.9305114746094, + "logps/rejected": -268.2242126464844, + "loss": 0.4153, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6965316534042358, + "rewards/margins": 1.7200047969818115, + "rewards/rejected": -2.4165360927581787, + "step": 310 + }, + { + "epoch": 0.33, + "learning_rate": 4.940384615384615e-07, + "logits/chosen": -3.1299521923065186, + "logits/rejected": -3.0084733963012695, + "logps/chosen": -251.54251098632812, + "logps/rejected": -237.68899536132812, + "loss": 0.3928, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.20220176875591278, + "rewards/margins": 1.855491042137146, + "rewards/rejected": -2.057692766189575, + "step": 320 + }, + { + "epoch": 0.34, + "learning_rate": 4.921153846153846e-07, + "logits/chosen": -3.0541279315948486, + "logits/rejected": -2.7398934364318848, + "logps/chosen": -311.959228515625, + "logps/rejected": -260.8400573730469, + "loss": 0.4465, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.2934505343437195, + "rewards/margins": 1.8156064748764038, + "rewards/rejected": -2.1090569496154785, + "step": 330 + }, + { + "epoch": 0.35, + "learning_rate": 4.901923076923077e-07, + "logits/chosen": -3.0184249877929688, + "logits/rejected": -2.7525899410247803, + "logps/chosen": -261.6094970703125, + "logps/rejected": -280.4756774902344, + "loss": 0.3781, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -0.4810101389884949, + "rewards/margins": 1.5501643419265747, + "rewards/rejected": -2.031174421310425, + "step": 340 + }, + { + "epoch": 0.36, + "learning_rate": 4.882692307692308e-07, + "logits/chosen": -2.9586663246154785, + "logits/rejected": -2.8382368087768555, + "logps/chosen": -298.5231628417969, + "logps/rejected": -261.07183837890625, + "loss": 0.4605, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.6761707067489624, + "rewards/margins": 1.5082601308822632, + "rewards/rejected": -2.1844308376312256, + "step": 350 + }, + { + "epoch": 0.37, + "learning_rate": 4.863461538461539e-07, + "logits/chosen": -3.0447680950164795, + "logits/rejected": -2.9561750888824463, + "logps/chosen": -325.26361083984375, + "logps/rejected": -243.9521026611328, + "loss": 0.3982, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.15374720096588135, + "rewards/margins": 2.1689236164093018, + "rewards/rejected": -2.3226709365844727, + "step": 360 + }, + { + "epoch": 0.38, + "learning_rate": 4.844230769230768e-07, + "logits/chosen": -3.171320676803589, + "logits/rejected": -2.795593023300171, + "logps/chosen": -270.4403076171875, + "logps/rejected": -263.0525207519531, + "loss": 0.4046, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.24999003112316132, + "rewards/margins": 1.4972503185272217, + "rewards/rejected": -1.7472403049468994, + "step": 370 + }, + { + "epoch": 0.39, + "learning_rate": 4.824999999999999e-07, + "logits/chosen": -3.139816999435425, + "logits/rejected": -2.8237149715423584, + "logps/chosen": -276.6899719238281, + "logps/rejected": -280.1494445800781, + "loss": 0.3929, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.36418357491493225, + "rewards/margins": 2.0872561931610107, + "rewards/rejected": -2.45143985748291, + "step": 380 + }, + { + "epoch": 0.4, + "learning_rate": 4.80576923076923e-07, + "logits/chosen": -3.0150628089904785, + "logits/rejected": -2.856785535812378, + "logps/chosen": -306.0409240722656, + "logps/rejected": -272.7051086425781, + "loss": 0.3961, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.2751230001449585, + "rewards/margins": 2.2792677879333496, + "rewards/rejected": -2.5543906688690186, + "step": 390 + }, + { + "epoch": 0.42, + "learning_rate": 4.786538461538461e-07, + "logits/chosen": -3.14699649810791, + "logits/rejected": -2.9324729442596436, + "logps/chosen": -299.39520263671875, + "logps/rejected": -281.36590576171875, + "loss": 0.3777, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.29352888464927673, + "rewards/margins": 2.5090556144714355, + "rewards/rejected": -2.802584171295166, + "step": 400 + }, + { + "epoch": 0.42, + "eval_logits/chosen": -3.047362804412842, + "eval_logits/rejected": -2.747648239135742, + "eval_logps/chosen": -307.6138000488281, + "eval_logps/rejected": -267.9804992675781, + "eval_loss": 0.40908658504486084, + "eval_rewards/accuracies": 0.7698412537574768, + "eval_rewards/chosen": -0.8597152233123779, + "eval_rewards/margins": 1.67086660861969, + "eval_rewards/rejected": -2.5305819511413574, + "eval_runtime": 130.4183, + "eval_samples_per_second": 15.335, + "eval_steps_per_second": 0.483, + "step": 400 + }, + { + "epoch": 0.43, + "learning_rate": 4.767307692307692e-07, + "logits/chosen": -3.156327724456787, + "logits/rejected": -2.8549814224243164, + "logps/chosen": -341.67694091796875, + "logps/rejected": -262.5184326171875, + "loss": 0.3913, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.92992103099823, + "rewards/margins": 1.826056718826294, + "rewards/rejected": -2.7559778690338135, + "step": 410 + }, + { + "epoch": 0.44, + "learning_rate": 4.748076923076923e-07, + "logits/chosen": -3.062770366668701, + "logits/rejected": -2.763805389404297, + "logps/chosen": -311.7411193847656, + "logps/rejected": -281.5027770996094, + "loss": 0.3828, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.719129204750061, + "rewards/margins": 2.130154848098755, + "rewards/rejected": -2.8492841720581055, + "step": 420 + }, + { + "epoch": 0.45, + "learning_rate": 4.7288461538461537e-07, + "logits/chosen": -2.8859896659851074, + "logits/rejected": -2.65274715423584, + "logps/chosen": -270.04840087890625, + "logps/rejected": -266.0298156738281, + "loss": 0.3871, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.6378560662269592, + "rewards/margins": 2.142416477203369, + "rewards/rejected": -2.7802722454071045, + "step": 430 + }, + { + "epoch": 0.46, + "learning_rate": 4.7096153846153846e-07, + "logits/chosen": -2.948652982711792, + "logits/rejected": -2.773026704788208, + "logps/chosen": -315.21234130859375, + "logps/rejected": -303.20916748046875, + "loss": 0.4083, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.8244800567626953, + "rewards/margins": 1.7506906986236572, + "rewards/rejected": -2.5751707553863525, + "step": 440 + }, + { + "epoch": 0.47, + "learning_rate": 4.690384615384615e-07, + "logits/chosen": -2.9108870029449463, + "logits/rejected": -2.7398343086242676, + "logps/chosen": -329.90936279296875, + "logps/rejected": -259.4378967285156, + "loss": 0.3518, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.48125535249710083, + "rewards/margins": 1.989284873008728, + "rewards/rejected": -2.4705405235290527, + "step": 450 + }, + { + "epoch": 0.48, + "learning_rate": 4.671153846153846e-07, + "logits/chosen": -2.8274548053741455, + "logits/rejected": -2.5006003379821777, + "logps/chosen": -306.48260498046875, + "logps/rejected": -256.0704650878906, + "loss": 0.3857, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.5667982697486877, + "rewards/margins": 1.9854114055633545, + "rewards/rejected": -2.5522096157073975, + "step": 460 + }, + { + "epoch": 0.49, + "learning_rate": 4.6519230769230766e-07, + "logits/chosen": -2.7966976165771484, + "logits/rejected": -2.4974465370178223, + "logps/chosen": -282.8033752441406, + "logps/rejected": -232.50863647460938, + "loss": 0.3525, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.0608681440353394, + "rewards/margins": 2.071587085723877, + "rewards/rejected": -3.132455587387085, + "step": 470 + }, + { + "epoch": 0.5, + "learning_rate": 4.6326923076923075e-07, + "logits/chosen": -2.6855077743530273, + "logits/rejected": -2.5066957473754883, + "logps/chosen": -252.94363403320312, + "logps/rejected": -242.0689239501953, + "loss": 0.3789, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -1.081735372543335, + "rewards/margins": 2.2696421146392822, + "rewards/rejected": -3.3513779640197754, + "step": 480 + }, + { + "epoch": 0.51, + "learning_rate": 4.6134615384615384e-07, + "logits/chosen": -2.599562168121338, + "logits/rejected": -2.308093309402466, + "logps/chosen": -275.1469421386719, + "logps/rejected": -234.97103881835938, + "loss": 0.3978, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.6643257141113281, + "rewards/margins": 2.531264543533325, + "rewards/rejected": -3.1955904960632324, + "step": 490 + }, + { + "epoch": 0.52, + "learning_rate": 4.5942307692307693e-07, + "logits/chosen": -2.674797296524048, + "logits/rejected": -2.2434916496276855, + "logps/chosen": -343.3860778808594, + "logps/rejected": -267.349365234375, + "loss": 0.3559, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.19713124632835388, + "rewards/margins": 2.5318844318389893, + "rewards/rejected": -2.729015827178955, + "step": 500 + }, + { + "epoch": 0.52, + "eval_logits/chosen": -2.6106228828430176, + "eval_logits/rejected": -2.2959861755371094, + "eval_logps/chosen": -309.4405822753906, + "eval_logps/rejected": -268.6939392089844, + "eval_loss": 0.433157354593277, + "eval_rewards/accuracies": 0.761904776096344, + "eval_rewards/chosen": -1.0423911809921265, + "eval_rewards/margins": 1.5595366954803467, + "eval_rewards/rejected": -2.6019277572631836, + "eval_runtime": 130.5063, + "eval_samples_per_second": 15.325, + "eval_steps_per_second": 0.483, + "step": 500 + }, + { + "epoch": 0.53, + "learning_rate": 4.575e-07, + "logits/chosen": -2.6816911697387695, + "logits/rejected": -2.3776135444641113, + "logps/chosen": -293.3232421875, + "logps/rejected": -270.6710205078125, + "loss": 0.3898, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.5901311039924622, + "rewards/margins": 3.1991240978240967, + "rewards/rejected": -3.7892556190490723, + "step": 510 + }, + { + "epoch": 0.54, + "learning_rate": 4.555769230769231e-07, + "logits/chosen": -2.821402072906494, + "logits/rejected": -2.4265995025634766, + "logps/chosen": -340.00909423828125, + "logps/rejected": -276.14141845703125, + "loss": 0.4053, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -1.2977416515350342, + "rewards/margins": 1.781804084777832, + "rewards/rejected": -3.079545736312866, + "step": 520 + }, + { + "epoch": 0.55, + "learning_rate": 4.5365384615384614e-07, + "logits/chosen": -2.635298252105713, + "logits/rejected": -2.775653123855591, + "logps/chosen": -278.55889892578125, + "logps/rejected": -281.57659912109375, + "loss": 0.403, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -1.3091782331466675, + "rewards/margins": 1.6853997707366943, + "rewards/rejected": -2.9945781230926514, + "step": 530 + }, + { + "epoch": 0.56, + "learning_rate": 4.5173076923076917e-07, + "logits/chosen": -2.858550548553467, + "logits/rejected": -2.5678272247314453, + "logps/chosen": -290.9246826171875, + "logps/rejected": -272.06829833984375, + "loss": 0.3998, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.6937220096588135, + "rewards/margins": 2.3342771530151367, + "rewards/rejected": -3.02799916267395, + "step": 540 + }, + { + "epoch": 0.57, + "learning_rate": 4.4980769230769226e-07, + "logits/chosen": -2.722504138946533, + "logits/rejected": -2.6339023113250732, + "logps/chosen": -322.2853088378906, + "logps/rejected": -271.86431884765625, + "loss": 0.3568, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.8023670315742493, + "rewards/margins": 2.3775863647460938, + "rewards/rejected": -3.1799533367156982, + "step": 550 + }, + { + "epoch": 0.58, + "learning_rate": 4.4788461538461534e-07, + "logits/chosen": -2.910511016845703, + "logits/rejected": -2.699862003326416, + "logps/chosen": -331.2922668457031, + "logps/rejected": -292.11846923828125, + "loss": 0.4005, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.6938377618789673, + "rewards/margins": 2.184812068939209, + "rewards/rejected": -2.8786497116088867, + "step": 560 + }, + { + "epoch": 0.59, + "learning_rate": 4.4596153846153843e-07, + "logits/chosen": -2.787506580352783, + "logits/rejected": -2.5669870376586914, + "logps/chosen": -309.0074157714844, + "logps/rejected": -302.73089599609375, + "loss": 0.3949, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.7182657122612, + "rewards/margins": 2.3996009826660156, + "rewards/rejected": -3.1178669929504395, + "step": 570 + }, + { + "epoch": 0.6, + "learning_rate": 4.440384615384615e-07, + "logits/chosen": -2.7322592735290527, + "logits/rejected": -2.562199831008911, + "logps/chosen": -312.8968505859375, + "logps/rejected": -307.6165771484375, + "loss": 0.3882, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.5363451838493347, + "rewards/margins": 2.4500136375427246, + "rewards/rejected": -2.986358880996704, + "step": 580 + }, + { + "epoch": 0.61, + "learning_rate": 4.421153846153846e-07, + "logits/chosen": -2.904258966445923, + "logits/rejected": -2.672128200531006, + "logps/chosen": -315.80322265625, + "logps/rejected": -271.91796875, + "loss": 0.3818, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.42090076208114624, + "rewards/margins": 2.514308214187622, + "rewards/rejected": -2.935209035873413, + "step": 590 + }, + { + "epoch": 0.62, + "learning_rate": 4.401923076923077e-07, + "logits/chosen": -2.841974973678589, + "logits/rejected": -2.6603472232818604, + "logps/chosen": -285.8795166015625, + "logps/rejected": -252.70083618164062, + "loss": 0.4178, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.4464953541755676, + "rewards/margins": 2.317780017852783, + "rewards/rejected": -2.764275550842285, + "step": 600 + }, + { + "epoch": 0.62, + "eval_logits/chosen": -2.8508059978485107, + "eval_logits/rejected": -2.5486865043640137, + "eval_logps/chosen": -305.4503173828125, + "eval_logps/rejected": -267.51214599609375, + "eval_loss": 0.3934193253517151, + "eval_rewards/accuracies": 0.7658730149269104, + "eval_rewards/chosen": -0.6433631181716919, + "eval_rewards/margins": 1.84038507938385, + "eval_rewards/rejected": -2.483747959136963, + "eval_runtime": 130.023, + "eval_samples_per_second": 15.382, + "eval_steps_per_second": 0.485, + "step": 600 + }, + { + "epoch": 0.63, + "learning_rate": 4.3826923076923073e-07, + "logits/chosen": -2.816254138946533, + "logits/rejected": -2.6248648166656494, + "logps/chosen": -279.0022888183594, + "logps/rejected": -250.00253295898438, + "loss": 0.3416, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -0.754069447517395, + "rewards/margins": 2.3438994884490967, + "rewards/rejected": -3.0979690551757812, + "step": 610 + }, + { + "epoch": 0.64, + "learning_rate": 4.363461538461538e-07, + "logits/chosen": -2.8594422340393066, + "logits/rejected": -2.776729106903076, + "logps/chosen": -272.91192626953125, + "logps/rejected": -247.15420532226562, + "loss": 0.3795, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.9311884045600891, + "rewards/margins": 2.104356050491333, + "rewards/rejected": -3.0355446338653564, + "step": 620 + }, + { + "epoch": 0.65, + "learning_rate": 4.344230769230769e-07, + "logits/chosen": -2.8708813190460205, + "logits/rejected": -2.5840649604797363, + "logps/chosen": -281.8168029785156, + "logps/rejected": -272.81683349609375, + "loss": 0.4112, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.6721941828727722, + "rewards/margins": 2.349375009536743, + "rewards/rejected": -3.021569013595581, + "step": 630 + }, + { + "epoch": 0.66, + "learning_rate": 4.325e-07, + "logits/chosen": -2.9108986854553223, + "logits/rejected": -2.547652006149292, + "logps/chosen": -326.2723083496094, + "logps/rejected": -333.6148986816406, + "loss": 0.3849, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.6807237863540649, + "rewards/margins": 1.8231480121612549, + "rewards/rejected": -2.5038719177246094, + "step": 640 + }, + { + "epoch": 0.67, + "learning_rate": 4.305769230769231e-07, + "logits/chosen": -2.7768750190734863, + "logits/rejected": -2.7807576656341553, + "logps/chosen": -287.76812744140625, + "logps/rejected": -309.85418701171875, + "loss": 0.3734, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5657386183738708, + "rewards/margins": 2.2681524753570557, + "rewards/rejected": -2.8338911533355713, + "step": 650 + }, + { + "epoch": 0.69, + "learning_rate": 4.2865384615384616e-07, + "logits/chosen": -2.9382200241088867, + "logits/rejected": -2.714677333831787, + "logps/chosen": -281.4496154785156, + "logps/rejected": -280.92034912109375, + "loss": 0.4074, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.7575949430465698, + "rewards/margins": 2.220801830291748, + "rewards/rejected": -2.9783966541290283, + "step": 660 + }, + { + "epoch": 0.7, + "learning_rate": 4.2673076923076925e-07, + "logits/chosen": -2.9047207832336426, + "logits/rejected": -2.573913097381592, + "logps/chosen": -262.4695739746094, + "logps/rejected": -235.21728515625, + "loss": 0.3538, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.9275886416435242, + "rewards/margins": 1.9407182931900024, + "rewards/rejected": -2.8683066368103027, + "step": 670 + }, + { + "epoch": 0.71, + "learning_rate": 4.2480769230769234e-07, + "logits/chosen": -2.900038242340088, + "logits/rejected": -2.5582680702209473, + "logps/chosen": -276.6561584472656, + "logps/rejected": -269.5938415527344, + "loss": 0.3675, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.9780898094177246, + "rewards/margins": 2.3385136127471924, + "rewards/rejected": -3.316603422164917, + "step": 680 + }, + { + "epoch": 0.72, + "learning_rate": 4.228846153846153e-07, + "logits/chosen": -2.9137685298919678, + "logits/rejected": -2.5705721378326416, + "logps/chosen": -312.13092041015625, + "logps/rejected": -296.21697998046875, + "loss": 0.41, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -1.045695424079895, + "rewards/margins": 2.037189245223999, + "rewards/rejected": -3.0828847885131836, + "step": 690 + }, + { + "epoch": 0.73, + "learning_rate": 4.209615384615384e-07, + "logits/chosen": -2.788215398788452, + "logits/rejected": -2.469395160675049, + "logps/chosen": -270.2633056640625, + "logps/rejected": -278.1541748046875, + "loss": 0.4206, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -1.6475088596343994, + "rewards/margins": 2.1269774436950684, + "rewards/rejected": -3.7744860649108887, + "step": 700 + }, + { + "epoch": 0.73, + "eval_logits/chosen": -2.87266206741333, + "eval_logits/rejected": -2.5678770542144775, + "eval_logps/chosen": -313.7168273925781, + "eval_logps/rejected": -277.7876892089844, + "eval_loss": 0.4058150053024292, + "eval_rewards/accuracies": 0.7857142686843872, + "eval_rewards/chosen": -1.4700164794921875, + "eval_rewards/margins": 2.041287660598755, + "eval_rewards/rejected": -3.5113041400909424, + "eval_runtime": 130.4287, + "eval_samples_per_second": 15.334, + "eval_steps_per_second": 0.483, + "step": 700 + }, + { + "epoch": 0.74, + "learning_rate": 4.190384615384615e-07, + "logits/chosen": -2.757584810256958, + "logits/rejected": -2.639995574951172, + "logps/chosen": -309.24932861328125, + "logps/rejected": -275.01385498046875, + "loss": 0.3832, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -1.292989730834961, + "rewards/margins": 2.1771950721740723, + "rewards/rejected": -3.4701850414276123, + "step": 710 + }, + { + "epoch": 0.75, + "learning_rate": 4.171153846153846e-07, + "logits/chosen": -2.867556571960449, + "logits/rejected": -2.7541584968566895, + "logps/chosen": -289.6522521972656, + "logps/rejected": -270.7252197265625, + "loss": 0.374, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -1.1409839391708374, + "rewards/margins": 2.2342498302459717, + "rewards/rejected": -3.3752338886260986, + "step": 720 + }, + { + "epoch": 0.76, + "learning_rate": 4.1519230769230767e-07, + "logits/chosen": -2.81394362449646, + "logits/rejected": -2.7436118125915527, + "logps/chosen": -259.9161682128906, + "logps/rejected": -304.9470520019531, + "loss": 0.3842, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": -1.0830919742584229, + "rewards/margins": 2.1413207054138184, + "rewards/rejected": -3.224412441253662, + "step": 730 + }, + { + "epoch": 0.77, + "learning_rate": 4.1326923076923075e-07, + "logits/chosen": -2.816218852996826, + "logits/rejected": -2.8035120964050293, + "logps/chosen": -329.23004150390625, + "logps/rejected": -238.5620574951172, + "loss": 0.4372, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.5688218474388123, + "rewards/margins": 2.4970219135284424, + "rewards/rejected": -3.0658438205718994, + "step": 740 + }, + { + "epoch": 0.78, + "learning_rate": 4.1134615384615384e-07, + "logits/chosen": -2.889146089553833, + "logits/rejected": -2.6036853790283203, + "logps/chosen": -312.89276123046875, + "logps/rejected": -251.8223114013672, + "loss": 0.3276, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.1251590251922607, + "rewards/margins": 2.237091302871704, + "rewards/rejected": -3.3622500896453857, + "step": 750 + }, + { + "epoch": 0.79, + "learning_rate": 4.0942307692307693e-07, + "logits/chosen": -2.934772491455078, + "logits/rejected": -2.5234997272491455, + "logps/chosen": -303.24603271484375, + "logps/rejected": -275.3195495605469, + "loss": 0.3925, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": -1.4970028400421143, + "rewards/margins": 2.127586603164673, + "rewards/rejected": -3.624589443206787, + "step": 760 + }, + { + "epoch": 0.8, + "learning_rate": 4.0749999999999996e-07, + "logits/chosen": -3.005864143371582, + "logits/rejected": -2.7517616748809814, + "logps/chosen": -339.569091796875, + "logps/rejected": -289.150146484375, + "loss": 0.3449, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -1.193687081336975, + "rewards/margins": 2.4689764976501465, + "rewards/rejected": -3.662663221359253, + "step": 770 + }, + { + "epoch": 0.81, + "learning_rate": 4.0557692307692305e-07, + "logits/chosen": -2.9409213066101074, + "logits/rejected": -2.467923402786255, + "logps/chosen": -321.47149658203125, + "logps/rejected": -257.263427734375, + "loss": 0.3687, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.8053227663040161, + "rewards/margins": 2.886740207672119, + "rewards/rejected": -3.692063093185425, + "step": 780 + }, + { + "epoch": 0.82, + "learning_rate": 4.0365384615384614e-07, + "logits/chosen": -3.0648276805877686, + "logits/rejected": -2.4790384769439697, + "logps/chosen": -272.0107116699219, + "logps/rejected": -259.4451904296875, + "loss": 0.4511, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -1.2784092426300049, + "rewards/margins": 2.352234363555908, + "rewards/rejected": -3.630643367767334, + "step": 790 + }, + { + "epoch": 0.83, + "learning_rate": 4.017307692307692e-07, + "logits/chosen": -3.0128531455993652, + "logits/rejected": -2.857410192489624, + "logps/chosen": -282.149169921875, + "logps/rejected": -283.2014465332031, + "loss": 0.4323, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.9385299682617188, + "rewards/margins": 1.8846447467803955, + "rewards/rejected": -2.823174476623535, + "step": 800 + }, + { + "epoch": 0.83, + "eval_logits/chosen": -2.9202206134796143, + "eval_logits/rejected": -2.6212680339813232, + "eval_logps/chosen": -308.0413818359375, + "eval_logps/rejected": -269.6094970703125, + "eval_loss": 0.3928997218608856, + "eval_rewards/accuracies": 0.7896825671195984, + "eval_rewards/chosen": -0.902471661567688, + "eval_rewards/margins": 1.7910076379776, + "eval_rewards/rejected": -2.693479537963867, + "eval_runtime": 130.5976, + "eval_samples_per_second": 15.314, + "eval_steps_per_second": 0.482, + "step": 800 + }, + { + "epoch": 0.84, + "learning_rate": 3.998076923076923e-07, + "logits/chosen": -2.789571762084961, + "logits/rejected": -2.89715838432312, + "logps/chosen": -284.51275634765625, + "logps/rejected": -245.26925659179688, + "loss": 0.4012, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": -0.8691709637641907, + "rewards/margins": 1.9500411748886108, + "rewards/rejected": -2.8192124366760254, + "step": 810 + }, + { + "epoch": 0.85, + "learning_rate": 3.978846153846154e-07, + "logits/chosen": -2.89267635345459, + "logits/rejected": -2.7063088417053223, + "logps/chosen": -335.8846740722656, + "logps/rejected": -306.52471923828125, + "loss": 0.3602, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.0393882989883423, + "rewards/margins": 1.985988974571228, + "rewards/rejected": -3.0253777503967285, + "step": 820 + }, + { + "epoch": 0.86, + "learning_rate": 3.9596153846153843e-07, + "logits/chosen": -2.7478280067443848, + "logits/rejected": -2.5870022773742676, + "logps/chosen": -260.309326171875, + "logps/rejected": -247.5229034423828, + "loss": 0.3506, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -0.8650934100151062, + "rewards/margins": 2.7803313732147217, + "rewards/rejected": -3.6454246044158936, + "step": 830 + }, + { + "epoch": 0.87, + "learning_rate": 3.940384615384615e-07, + "logits/chosen": -2.7750461101531982, + "logits/rejected": -2.84771990776062, + "logps/chosen": -300.90875244140625, + "logps/rejected": -278.62127685546875, + "loss": 0.3679, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.6639699339866638, + "rewards/margins": 2.6008620262145996, + "rewards/rejected": -3.264832019805908, + "step": 840 + }, + { + "epoch": 0.88, + "learning_rate": 3.9211538461538455e-07, + "logits/chosen": -3.0167763233184814, + "logits/rejected": -2.3664677143096924, + "logps/chosen": -319.5389709472656, + "logps/rejected": -274.7938537597656, + "loss": 0.3992, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.3606072664260864, + "rewards/margins": 2.127211093902588, + "rewards/rejected": -3.4878182411193848, + "step": 850 + }, + { + "epoch": 0.89, + "learning_rate": 3.9019230769230764e-07, + "logits/chosen": -2.848731279373169, + "logits/rejected": -2.658998489379883, + "logps/chosen": -264.0252380371094, + "logps/rejected": -246.7176971435547, + "loss": 0.4053, + "rewards/accuracies": 0.8125, + "rewards/chosen": -1.7886860370635986, + "rewards/margins": 2.0003931522369385, + "rewards/rejected": -3.789078950881958, + "step": 860 + }, + { + "epoch": 0.9, + "learning_rate": 3.8826923076923073e-07, + "logits/chosen": -2.9210238456726074, + "logits/rejected": -2.798243284225464, + "logps/chosen": -290.9345703125, + "logps/rejected": -271.5350646972656, + "loss": 0.3502, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": -1.4627825021743774, + "rewards/margins": 1.9910736083984375, + "rewards/rejected": -3.4538562297821045, + "step": 870 + }, + { + "epoch": 0.91, + "learning_rate": 3.863461538461538e-07, + "logits/chosen": -2.922494411468506, + "logits/rejected": -2.602292776107788, + "logps/chosen": -267.75848388671875, + "logps/rejected": -272.4598693847656, + "loss": 0.4225, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -1.7402254343032837, + "rewards/margins": 2.069711208343506, + "rewards/rejected": -3.809937000274658, + "step": 880 + }, + { + "epoch": 0.92, + "learning_rate": 3.844230769230769e-07, + "logits/chosen": -2.877516269683838, + "logits/rejected": -2.51607084274292, + "logps/chosen": -338.27886962890625, + "logps/rejected": -259.23712158203125, + "loss": 0.3949, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -1.178595781326294, + "rewards/margins": 2.4852442741394043, + "rewards/rejected": -3.663839817047119, + "step": 890 + }, + { + "epoch": 0.93, + "learning_rate": 3.825e-07, + "logits/chosen": -2.664858102798462, + "logits/rejected": -2.485161542892456, + "logps/chosen": -310.84686279296875, + "logps/rejected": -259.01397705078125, + "loss": 0.3706, + "rewards/accuracies": 0.875, + "rewards/chosen": -1.098044514656067, + "rewards/margins": 2.7692551612854004, + "rewards/rejected": -3.8672993183135986, + "step": 900 + }, + { + "epoch": 0.93, + "eval_logits/chosen": -2.8415608406066895, + "eval_logits/rejected": -2.542830467224121, + "eval_logps/chosen": -310.1387939453125, + "eval_logps/rejected": -272.931640625, + "eval_loss": 0.3902662396430969, + "eval_rewards/accuracies": 0.8055555820465088, + "eval_rewards/chosen": -1.1122112274169922, + "eval_rewards/margins": 1.9134843349456787, + "eval_rewards/rejected": -3.025695562362671, + "eval_runtime": 130.5238, + "eval_samples_per_second": 15.323, + "eval_steps_per_second": 0.483, + "step": 900 + }, + { + "epoch": 0.94, + "learning_rate": 3.805769230769231e-07, + "logits/chosen": -2.7821085453033447, + "logits/rejected": -2.519225597381592, + "logps/chosen": -279.53466796875, + "logps/rejected": -229.2816619873047, + "loss": 0.3435, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.7595919370651245, + "rewards/margins": 2.5894370079040527, + "rewards/rejected": -3.3490288257598877, + "step": 910 + }, + { + "epoch": 0.96, + "learning_rate": 3.7865384615384617e-07, + "logits/chosen": -3.0091214179992676, + "logits/rejected": -2.694118022918701, + "logps/chosen": -327.59515380859375, + "logps/rejected": -311.08660888671875, + "loss": 0.3721, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.9237958788871765, + "rewards/margins": 2.2765049934387207, + "rewards/rejected": -3.200300693511963, + "step": 920 + }, + { + "epoch": 0.97, + "learning_rate": 3.7673076923076925e-07, + "logits/chosen": -2.812438488006592, + "logits/rejected": -2.4150969982147217, + "logps/chosen": -339.35986328125, + "logps/rejected": -292.71905517578125, + "loss": 0.3689, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.5981228947639465, + "rewards/margins": 3.090789318084717, + "rewards/rejected": -3.6889121532440186, + "step": 930 + }, + { + "epoch": 0.98, + "learning_rate": 3.748076923076923e-07, + "logits/chosen": -2.8114781379699707, + "logits/rejected": -2.5097975730895996, + "logps/chosen": -309.5245666503906, + "logps/rejected": -285.4954833984375, + "loss": 0.3759, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": -0.9724052548408508, + "rewards/margins": 2.3283073902130127, + "rewards/rejected": -3.3007121086120605, + "step": 940 + }, + { + "epoch": 0.99, + "learning_rate": 3.728846153846154e-07, + "logits/chosen": -2.870340585708618, + "logits/rejected": -2.547656536102295, + "logps/chosen": -358.37042236328125, + "logps/rejected": -298.05047607421875, + "loss": 0.3706, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": -1.174553394317627, + "rewards/margins": 2.58784818649292, + "rewards/rejected": -3.762402057647705, + "step": 950 + }, + { + "epoch": 1.0, + "learning_rate": 3.7096153846153846e-07, + "logits/chosen": -2.8550620079040527, + "logits/rejected": -2.5932936668395996, + "logps/chosen": -332.6602478027344, + "logps/rejected": -313.8180236816406, + "loss": 0.3257, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.8856698274612427, + "rewards/margins": 2.5121753215789795, + "rewards/rejected": -3.3978450298309326, + "step": 960 + }, + { + "epoch": 1.01, + "learning_rate": 3.690384615384615e-07, + "logits/chosen": -2.858963966369629, + "logits/rejected": -2.5064713954925537, + "logps/chosen": -332.31402587890625, + "logps/rejected": -266.08477783203125, + "loss": 0.1583, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -0.8705652952194214, + "rewards/margins": 4.396849632263184, + "rewards/rejected": -5.2674150466918945, + "step": 970 + }, + { + "epoch": 1.02, + "learning_rate": 3.671153846153846e-07, + "logits/chosen": -2.7196831703186035, + "logits/rejected": -2.562572717666626, + "logps/chosen": -274.79473876953125, + "logps/rejected": -267.02587890625, + "loss": 0.0551, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.3285406231880188, + "rewards/margins": 5.019989013671875, + "rewards/rejected": -5.348529815673828, + "step": 980 + }, + { + "epoch": 1.03, + "learning_rate": 3.6519230769230767e-07, + "logits/chosen": -2.827141046524048, + "logits/rejected": -2.4708828926086426, + "logps/chosen": -259.7366027832031, + "logps/rejected": -272.045654296875, + "loss": 0.0773, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 0.058406610041856766, + "rewards/margins": 5.3060407638549805, + "rewards/rejected": -5.247633934020996, + "step": 990 + }, + { + "epoch": 1.04, + "learning_rate": 3.6326923076923076e-07, + "logits/chosen": -2.8863983154296875, + "logits/rejected": -2.680694341659546, + "logps/chosen": -307.0455322265625, + "logps/rejected": -273.3485107421875, + "loss": 0.0496, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.03276168182492256, + "rewards/margins": 6.467474937438965, + "rewards/rejected": -6.500236511230469, + "step": 1000 + }, + { + "epoch": 1.04, + "eval_logits/chosen": -2.8150267601013184, + "eval_logits/rejected": -2.509260654449463, + "eval_logps/chosen": -313.2651062011719, + "eval_logps/rejected": -283.91961669921875, + "eval_loss": 0.3991049826145172, + "eval_rewards/accuracies": 0.8015872836112976, + "eval_rewards/chosen": -1.4248411655426025, + "eval_rewards/margins": 2.699652671813965, + "eval_rewards/rejected": -4.1244940757751465, + "eval_runtime": 137.8475, + "eval_samples_per_second": 14.509, + "eval_steps_per_second": 0.457, + "step": 1000 + }, + { + "epoch": 1.05, + "learning_rate": 3.6134615384615384e-07, + "logits/chosen": -2.7880311012268066, + "logits/rejected": -2.579087018966675, + "logps/chosen": -271.03375244140625, + "logps/rejected": -290.6522216796875, + "loss": 0.0675, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 0.3281337320804596, + "rewards/margins": 6.674640655517578, + "rewards/rejected": -6.3465070724487305, + "step": 1010 + }, + { + "epoch": 1.06, + "learning_rate": 3.594230769230769e-07, + "logits/chosen": -2.804231882095337, + "logits/rejected": -2.3508119583129883, + "logps/chosen": -286.9497375488281, + "logps/rejected": -285.6386413574219, + "loss": 0.0677, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.19059844315052032, + "rewards/margins": 5.62130880355835, + "rewards/rejected": -5.430710315704346, + "step": 1020 + }, + { + "epoch": 1.07, + "learning_rate": 3.5749999999999997e-07, + "logits/chosen": -2.8610713481903076, + "logits/rejected": -2.537271022796631, + "logps/chosen": -323.7139587402344, + "logps/rejected": -295.88519287109375, + "loss": 0.0585, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10528359562158585, + "rewards/margins": 6.13063907623291, + "rewards/rejected": -6.025355339050293, + "step": 1030 + }, + { + "epoch": 1.08, + "learning_rate": 3.5557692307692305e-07, + "logits/chosen": -2.777980327606201, + "logits/rejected": -2.4435582160949707, + "logps/chosen": -268.0248107910156, + "logps/rejected": -285.71185302734375, + "loss": 0.0522, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.2603774666786194, + "rewards/margins": 6.728521823883057, + "rewards/rejected": -6.988899230957031, + "step": 1040 + }, + { + "epoch": 1.09, + "learning_rate": 3.5365384615384614e-07, + "logits/chosen": -2.6917247772216797, + "logits/rejected": -2.6877942085266113, + "logps/chosen": -268.5338439941406, + "logps/rejected": -299.14337158203125, + "loss": 0.0544, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.6682311296463013, + "rewards/margins": 4.996738433837891, + "rewards/rejected": -5.664969444274902, + "step": 1050 + }, + { + "epoch": 1.1, + "learning_rate": 3.5173076923076923e-07, + "logits/chosen": -2.8935842514038086, + "logits/rejected": -2.620191812515259, + "logps/chosen": -360.71923828125, + "logps/rejected": -382.8384704589844, + "loss": 0.053, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.1445586234331131, + "rewards/margins": 6.801445007324219, + "rewards/rejected": -6.946004390716553, + "step": 1060 + }, + { + "epoch": 1.11, + "learning_rate": 3.498076923076923e-07, + "logits/chosen": -2.8603906631469727, + "logits/rejected": -2.4893665313720703, + "logps/chosen": -288.63330078125, + "logps/rejected": -276.2620849609375, + "loss": 0.0603, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 0.13872453570365906, + "rewards/margins": 5.943129539489746, + "rewards/rejected": -5.80440616607666, + "step": 1070 + }, + { + "epoch": 1.12, + "learning_rate": 3.478846153846154e-07, + "logits/chosen": -2.8175647258758545, + "logits/rejected": -2.600020408630371, + "logps/chosen": -303.94549560546875, + "logps/rejected": -291.47125244140625, + "loss": 0.0507, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 0.09348030388355255, + "rewards/margins": 6.168398857116699, + "rewards/rejected": -6.074917793273926, + "step": 1080 + }, + { + "epoch": 1.13, + "learning_rate": 3.459615384615385e-07, + "logits/chosen": -2.5639805793762207, + "logits/rejected": -2.6261074542999268, + "logps/chosen": -293.8769836425781, + "logps/rejected": -305.51483154296875, + "loss": 0.0556, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.5082187652587891, + "rewards/margins": 5.773597240447998, + "rewards/rejected": -6.281815528869629, + "step": 1090 + }, + { + "epoch": 1.14, + "learning_rate": 3.440384615384615e-07, + "logits/chosen": -2.915066957473755, + "logits/rejected": -2.7044992446899414, + "logps/chosen": -264.509033203125, + "logps/rejected": -311.0338134765625, + "loss": 0.0723, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.5605603456497192, + "rewards/margins": 5.587393760681152, + "rewards/rejected": -6.147953510284424, + "step": 1100 + }, + { + "epoch": 1.14, + "eval_logits/chosen": -2.8242454528808594, + "eval_logits/rejected": -2.5170044898986816, + "eval_logps/chosen": -317.8056335449219, + "eval_logps/rejected": -287.9914245605469, + "eval_loss": 0.3998846411705017, + "eval_rewards/accuracies": 0.7896825671195984, + "eval_rewards/chosen": -1.8788989782333374, + "eval_rewards/margins": 2.6527750492095947, + "eval_rewards/rejected": -4.531673908233643, + "eval_runtime": 138.2758, + "eval_samples_per_second": 14.464, + "eval_steps_per_second": 0.456, + "step": 1100 + }, + { + "epoch": 1.15, + "learning_rate": 3.421153846153846e-07, + "logits/chosen": -2.852841377258301, + "logits/rejected": -2.838223934173584, + "logps/chosen": -267.4649658203125, + "logps/rejected": -319.916015625, + "loss": 0.0478, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.17563840746879578, + "rewards/margins": 6.513575077056885, + "rewards/rejected": -6.689213752746582, + "step": 1110 + }, + { + "epoch": 1.16, + "learning_rate": 3.4019230769230764e-07, + "logits/chosen": -2.8269314765930176, + "logits/rejected": -2.4453513622283936, + "logps/chosen": -298.703857421875, + "logps/rejected": -329.9590759277344, + "loss": 0.0575, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.23525571823120117, + "rewards/margins": 6.723779201507568, + "rewards/rejected": -6.9590349197387695, + "step": 1120 + }, + { + "epoch": 1.17, + "learning_rate": 3.3826923076923073e-07, + "logits/chosen": -2.8692283630371094, + "logits/rejected": -2.5266168117523193, + "logps/chosen": -304.0444641113281, + "logps/rejected": -332.7533264160156, + "loss": 0.0816, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5301819443702698, + "rewards/margins": 6.649426460266113, + "rewards/rejected": -7.179608345031738, + "step": 1130 + }, + { + "epoch": 1.18, + "learning_rate": 3.363461538461538e-07, + "logits/chosen": -2.8497471809387207, + "logits/rejected": -2.7025351524353027, + "logps/chosen": -268.01116943359375, + "logps/rejected": -269.3169250488281, + "loss": 0.0541, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7645604610443115, + "rewards/margins": 5.691164970397949, + "rewards/rejected": -6.45572566986084, + "step": 1140 + }, + { + "epoch": 1.19, + "learning_rate": 3.344230769230769e-07, + "logits/chosen": -2.7669548988342285, + "logits/rejected": -2.582608699798584, + "logps/chosen": -297.6730651855469, + "logps/rejected": -308.69866943359375, + "loss": 0.0786, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.13068512082099915, + "rewards/margins": 6.759812355041504, + "rewards/rejected": -6.89049768447876, + "step": 1150 + }, + { + "epoch": 1.2, + "learning_rate": 3.325e-07, + "logits/chosen": -2.880467653274536, + "logits/rejected": -2.482970714569092, + "logps/chosen": -325.6810607910156, + "logps/rejected": -317.48223876953125, + "loss": 0.0557, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.18659517168998718, + "rewards/margins": 6.584499359130859, + "rewards/rejected": -6.77109432220459, + "step": 1160 + }, + { + "epoch": 1.21, + "learning_rate": 3.305769230769231e-07, + "logits/chosen": -2.910566806793213, + "logits/rejected": -2.390554904937744, + "logps/chosen": -335.02728271484375, + "logps/rejected": -297.4922180175781, + "loss": 0.0497, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.5198079347610474, + "rewards/margins": 6.664822578430176, + "rewards/rejected": -7.184630393981934, + "step": 1170 + }, + { + "epoch": 1.23, + "learning_rate": 3.286538461538461e-07, + "logits/chosen": -2.741680383682251, + "logits/rejected": -2.6448922157287598, + "logps/chosen": -293.5112609863281, + "logps/rejected": -307.81072998046875, + "loss": 0.0548, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -1.1880362033843994, + "rewards/margins": 6.250551700592041, + "rewards/rejected": -7.4385881423950195, + "step": 1180 + }, + { + "epoch": 1.24, + "learning_rate": 3.267307692307692e-07, + "logits/chosen": -2.8261971473693848, + "logits/rejected": -2.5925064086914062, + "logps/chosen": -341.7809143066406, + "logps/rejected": -322.70599365234375, + "loss": 0.0556, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.081176519393921, + "rewards/margins": 7.538055419921875, + "rewards/rejected": -8.619232177734375, + "step": 1190 + }, + { + "epoch": 1.25, + "learning_rate": 3.248076923076923e-07, + "logits/chosen": -2.762108325958252, + "logits/rejected": -2.593078136444092, + "logps/chosen": -305.1072692871094, + "logps/rejected": -306.4645690917969, + "loss": 0.0481, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -1.4623178243637085, + "rewards/margins": 5.997315406799316, + "rewards/rejected": -7.459633827209473, + "step": 1200 + }, + { + "epoch": 1.25, + "eval_logits/chosen": -2.810943603515625, + "eval_logits/rejected": -2.513878345489502, + "eval_logps/chosen": -325.2280578613281, + "eval_logps/rejected": -297.9687194824219, + "eval_loss": 0.41905704140663147, + "eval_rewards/accuracies": 0.7817460298538208, + "eval_rewards/chosen": -2.6211390495300293, + "eval_rewards/margins": 2.90826416015625, + "eval_rewards/rejected": -5.529403209686279, + "eval_runtime": 138.4959, + "eval_samples_per_second": 14.441, + "eval_steps_per_second": 0.455, + "step": 1200 + }, + { + "epoch": 1.26, + "learning_rate": 3.228846153846154e-07, + "logits/chosen": -2.7453770637512207, + "logits/rejected": -2.585190773010254, + "logps/chosen": -299.1862487792969, + "logps/rejected": -337.72320556640625, + "loss": 0.0456, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.7499379515647888, + "rewards/margins": 6.9477972984313965, + "rewards/rejected": -7.697734832763672, + "step": 1210 + }, + { + "epoch": 1.27, + "learning_rate": 3.2096153846153846e-07, + "logits/chosen": -2.8571231365203857, + "logits/rejected": -2.7352118492126465, + "logps/chosen": -290.97100830078125, + "logps/rejected": -344.00579833984375, + "loss": 0.0631, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.5878090858459473, + "rewards/margins": 7.520246982574463, + "rewards/rejected": -8.10805606842041, + "step": 1220 + }, + { + "epoch": 1.28, + "learning_rate": 3.1903846153846155e-07, + "logits/chosen": -2.8437607288360596, + "logits/rejected": -2.3721210956573486, + "logps/chosen": -337.2201232910156, + "logps/rejected": -293.2413635253906, + "loss": 0.0778, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": -0.8207977414131165, + "rewards/margins": 6.183731555938721, + "rewards/rejected": -7.004528999328613, + "step": 1230 + }, + { + "epoch": 1.29, + "learning_rate": 3.1711538461538464e-07, + "logits/chosen": -2.7952797412872314, + "logits/rejected": -2.5198020935058594, + "logps/chosen": -278.1673889160156, + "logps/rejected": -294.428955078125, + "loss": 0.0575, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.46408623456954956, + "rewards/margins": 6.282520771026611, + "rewards/rejected": -6.746606349945068, + "step": 1240 + }, + { + "epoch": 1.3, + "learning_rate": 3.151923076923077e-07, + "logits/chosen": -2.8894572257995605, + "logits/rejected": -2.6253533363342285, + "logps/chosen": -316.9076843261719, + "logps/rejected": -325.4949035644531, + "loss": 0.0621, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 0.14270208775997162, + "rewards/margins": 6.009703636169434, + "rewards/rejected": -5.867001533508301, + "step": 1250 + }, + { + "epoch": 1.31, + "learning_rate": 3.132692307692307e-07, + "logits/chosen": -2.8078317642211914, + "logits/rejected": -2.5614686012268066, + "logps/chosen": -298.93621826171875, + "logps/rejected": -320.55828857421875, + "loss": 0.052, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.061225999146699905, + "rewards/margins": 6.401114463806152, + "rewards/rejected": -6.462340354919434, + "step": 1260 + }, + { + "epoch": 1.32, + "learning_rate": 3.113461538461538e-07, + "logits/chosen": -2.8679614067077637, + "logits/rejected": -2.5462114810943604, + "logps/chosen": -332.6659240722656, + "logps/rejected": -304.09649658203125, + "loss": 0.0415, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 0.08115784078836441, + "rewards/margins": 6.285240173339844, + "rewards/rejected": -6.204082012176514, + "step": 1270 + }, + { + "epoch": 1.33, + "learning_rate": 3.094230769230769e-07, + "logits/chosen": -2.8452160358428955, + "logits/rejected": -2.593921184539795, + "logps/chosen": -278.31298828125, + "logps/rejected": -268.8619689941406, + "loss": 0.051, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.3518837094306946, + "rewards/margins": 6.064053535461426, + "rewards/rejected": -6.4159369468688965, + "step": 1280 + }, + { + "epoch": 1.34, + "learning_rate": 3.0749999999999997e-07, + "logits/chosen": -2.8956944942474365, + "logits/rejected": -2.659951686859131, + "logps/chosen": -315.71759033203125, + "logps/rejected": -329.34552001953125, + "loss": 0.0662, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.39976391196250916, + "rewards/margins": 6.577657222747803, + "rewards/rejected": -6.977420806884766, + "step": 1290 + }, + { + "epoch": 1.35, + "learning_rate": 3.0557692307692305e-07, + "logits/chosen": -2.820831060409546, + "logits/rejected": -2.7252230644226074, + "logps/chosen": -275.63775634765625, + "logps/rejected": -314.5214538574219, + "loss": 0.0432, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7111703157424927, + "rewards/margins": 6.398756980895996, + "rewards/rejected": -7.109927177429199, + "step": 1300 + }, + { + "epoch": 1.35, + "eval_logits/chosen": -2.8120782375335693, + "eval_logits/rejected": -2.515331506729126, + "eval_logps/chosen": -319.6214294433594, + "eval_logps/rejected": -293.1344909667969, + "eval_loss": 0.40695273876190186, + "eval_rewards/accuracies": 0.8055555820465088, + "eval_rewards/chosen": -2.0604758262634277, + "eval_rewards/margins": 2.985503911972046, + "eval_rewards/rejected": -5.045979976654053, + "eval_runtime": 138.1372, + "eval_samples_per_second": 14.478, + "eval_steps_per_second": 0.456, + "step": 1300 + }, + { + "epoch": 1.36, + "learning_rate": 3.0365384615384614e-07, + "logits/chosen": -2.879749298095703, + "logits/rejected": -2.674367666244507, + "logps/chosen": -315.0393371582031, + "logps/rejected": -354.39971923828125, + "loss": 0.0577, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.607972264289856, + "rewards/margins": 6.823029518127441, + "rewards/rejected": -7.431002616882324, + "step": 1310 + }, + { + "epoch": 1.37, + "learning_rate": 3.0173076923076923e-07, + "logits/chosen": -2.8352997303009033, + "logits/rejected": -2.622284412384033, + "logps/chosen": -293.62371826171875, + "logps/rejected": -313.57244873046875, + "loss": 0.0372, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.08940573781728745, + "rewards/margins": 7.263546943664551, + "rewards/rejected": -7.352952003479004, + "step": 1320 + }, + { + "epoch": 1.38, + "learning_rate": 2.998076923076923e-07, + "logits/chosen": -2.805799961090088, + "logits/rejected": -2.57133150100708, + "logps/chosen": -323.6988220214844, + "logps/rejected": -278.4544677734375, + "loss": 0.0631, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.618446409702301, + "rewards/margins": 6.439663887023926, + "rewards/rejected": -7.058110237121582, + "step": 1330 + }, + { + "epoch": 1.39, + "learning_rate": 2.9788461538461535e-07, + "logits/chosen": -2.750667095184326, + "logits/rejected": -2.553912878036499, + "logps/chosen": -262.938232421875, + "logps/rejected": -265.05987548828125, + "loss": 0.057, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.6424404978752136, + "rewards/margins": 6.154696464538574, + "rewards/rejected": -6.7971367835998535, + "step": 1340 + }, + { + "epoch": 1.4, + "learning_rate": 2.9596153846153844e-07, + "logits/chosen": -2.74922776222229, + "logits/rejected": -2.522984743118286, + "logps/chosen": -296.0598449707031, + "logps/rejected": -295.513916015625, + "loss": 0.0725, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.9643446207046509, + "rewards/margins": 6.230860233306885, + "rewards/rejected": -7.195204734802246, + "step": 1350 + }, + { + "epoch": 1.41, + "learning_rate": 2.940384615384615e-07, + "logits/chosen": -2.760007381439209, + "logits/rejected": -2.3878226280212402, + "logps/chosen": -271.11651611328125, + "logps/rejected": -298.1424560546875, + "loss": 0.0646, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.8373205065727234, + "rewards/margins": 6.260037422180176, + "rewards/rejected": -7.097357749938965, + "step": 1360 + }, + { + "epoch": 1.42, + "learning_rate": 2.921153846153846e-07, + "logits/chosen": -2.7579071521759033, + "logits/rejected": -2.446126937866211, + "logps/chosen": -356.7608337402344, + "logps/rejected": -328.30108642578125, + "loss": 0.0579, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.24769195914268494, + "rewards/margins": 7.452914237976074, + "rewards/rejected": -7.700606346130371, + "step": 1370 + }, + { + "epoch": 1.43, + "learning_rate": 2.901923076923077e-07, + "logits/chosen": -2.6069226264953613, + "logits/rejected": -2.6243488788604736, + "logps/chosen": -317.1035461425781, + "logps/rejected": -319.14581298828125, + "loss": 0.0676, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.2916959822177887, + "rewards/margins": 6.393536567687988, + "rewards/rejected": -6.685232639312744, + "step": 1380 + }, + { + "epoch": 1.44, + "learning_rate": 2.882692307692308e-07, + "logits/chosen": -2.725607395172119, + "logits/rejected": -2.3066391944885254, + "logps/chosen": -319.43157958984375, + "logps/rejected": -315.57080078125, + "loss": 0.0658, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.6996710896492004, + "rewards/margins": 6.387073040008545, + "rewards/rejected": -7.0867438316345215, + "step": 1390 + }, + { + "epoch": 1.45, + "learning_rate": 2.863461538461539e-07, + "logits/chosen": -2.8061482906341553, + "logits/rejected": -2.6063475608825684, + "logps/chosen": -319.04803466796875, + "logps/rejected": -313.13494873046875, + "loss": 0.0402, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.17195379734039307, + "rewards/margins": 7.243444919586182, + "rewards/rejected": -7.41540002822876, + "step": 1400 + }, + { + "epoch": 1.45, + "eval_logits/chosen": -2.7388460636138916, + "eval_logits/rejected": -2.4383180141448975, + "eval_logps/chosen": -321.46142578125, + "eval_logps/rejected": -293.61639404296875, + "eval_loss": 0.400052547454834, + "eval_rewards/accuracies": 0.7936508059501648, + "eval_rewards/chosen": -2.244476795196533, + "eval_rewards/margins": 2.8496947288513184, + "eval_rewards/rejected": -5.094171047210693, + "eval_runtime": 137.973, + "eval_samples_per_second": 14.496, + "eval_steps_per_second": 0.457, + "step": 1400 + }, + { + "epoch": 1.46, + "learning_rate": 2.844230769230769e-07, + "logits/chosen": -2.73895001411438, + "logits/rejected": -2.6003918647766113, + "logps/chosen": -286.24420166015625, + "logps/rejected": -312.2796325683594, + "loss": 0.086, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7323522567749023, + "rewards/margins": 6.541958808898926, + "rewards/rejected": -7.274311065673828, + "step": 1410 + }, + { + "epoch": 1.47, + "learning_rate": 2.8249999999999994e-07, + "logits/chosen": -2.805447816848755, + "logits/rejected": -2.335646867752075, + "logps/chosen": -316.35821533203125, + "logps/rejected": -295.689453125, + "loss": 0.0389, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.7643243670463562, + "rewards/margins": 6.689816474914551, + "rewards/rejected": -7.454140663146973, + "step": 1420 + }, + { + "epoch": 1.48, + "learning_rate": 2.8057692307692303e-07, + "logits/chosen": -2.6269233226776123, + "logits/rejected": -2.334346294403076, + "logps/chosen": -292.73297119140625, + "logps/rejected": -307.1222229003906, + "loss": 0.0552, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.6479349732398987, + "rewards/margins": 6.831157684326172, + "rewards/rejected": -7.479092597961426, + "step": 1430 + }, + { + "epoch": 1.5, + "learning_rate": 2.786538461538461e-07, + "logits/chosen": -2.6912832260131836, + "logits/rejected": -2.435011625289917, + "logps/chosen": -319.80609130859375, + "logps/rejected": -309.6658630371094, + "loss": 0.0503, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.33509448170661926, + "rewards/margins": 7.094533443450928, + "rewards/rejected": -7.429627895355225, + "step": 1440 + }, + { + "epoch": 1.51, + "learning_rate": 2.767307692307692e-07, + "logits/chosen": -2.567253589630127, + "logits/rejected": -2.4747424125671387, + "logps/chosen": -294.86395263671875, + "logps/rejected": -295.82183837890625, + "loss": 0.0523, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.19534453749656677, + "rewards/margins": 6.722334861755371, + "rewards/rejected": -6.917679786682129, + "step": 1450 + }, + { + "epoch": 1.52, + "learning_rate": 2.748076923076923e-07, + "logits/chosen": -2.699418783187866, + "logits/rejected": -2.5728678703308105, + "logps/chosen": -329.30322265625, + "logps/rejected": -334.52044677734375, + "loss": 0.0589, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.20032933354377747, + "rewards/margins": 6.842965602874756, + "rewards/rejected": -7.043294429779053, + "step": 1460 + }, + { + "epoch": 1.53, + "learning_rate": 2.728846153846154e-07, + "logits/chosen": -2.674626588821411, + "logits/rejected": -2.435716152191162, + "logps/chosen": -337.5685729980469, + "logps/rejected": -332.3224182128906, + "loss": 0.0751, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.4591027796268463, + "rewards/margins": 6.688274383544922, + "rewards/rejected": -7.147377014160156, + "step": 1470 + }, + { + "epoch": 1.54, + "learning_rate": 2.7096153846153847e-07, + "logits/chosen": -2.704498767852783, + "logits/rejected": -2.261159896850586, + "logps/chosen": -288.1585693359375, + "logps/rejected": -305.39166259765625, + "loss": 0.0592, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.35019350051879883, + "rewards/margins": 6.569688320159912, + "rewards/rejected": -6.919882297515869, + "step": 1480 + }, + { + "epoch": 1.55, + "learning_rate": 2.6903846153846155e-07, + "logits/chosen": -2.7152137756347656, + "logits/rejected": -2.2341487407684326, + "logps/chosen": -273.33880615234375, + "logps/rejected": -277.2101745605469, + "loss": 0.0471, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.8817023038864136, + "rewards/margins": 5.90345573425293, + "rewards/rejected": -6.785158634185791, + "step": 1490 + }, + { + "epoch": 1.56, + "learning_rate": 2.671153846153846e-07, + "logits/chosen": -2.6211445331573486, + "logits/rejected": -2.372669219970703, + "logps/chosen": -328.92657470703125, + "logps/rejected": -282.3219299316406, + "loss": 0.0529, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.9412908554077148, + "rewards/margins": 6.654592990875244, + "rewards/rejected": -7.595883846282959, + "step": 1500 + }, + { + "epoch": 1.56, + "eval_logits/chosen": -2.696347236633301, + "eval_logits/rejected": -2.3906466960906982, + "eval_logps/chosen": -322.5152587890625, + "eval_logps/rejected": -295.14263916015625, + "eval_loss": 0.4066043198108673, + "eval_rewards/accuracies": 0.8015872836112976, + "eval_rewards/chosen": -2.3498613834381104, + "eval_rewards/margins": 2.8969383239746094, + "eval_rewards/rejected": -5.246799945831299, + "eval_runtime": 138.1634, + "eval_samples_per_second": 14.476, + "eval_steps_per_second": 0.456, + "step": 1500 + }, + { + "epoch": 1.57, + "learning_rate": 2.651923076923077e-07, + "logits/chosen": -2.6930763721466064, + "logits/rejected": -2.4558794498443604, + "logps/chosen": -297.29498291015625, + "logps/rejected": -310.0185546875, + "loss": 0.0558, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -1.2887471914291382, + "rewards/margins": 6.140270709991455, + "rewards/rejected": -7.429017543792725, + "step": 1510 + }, + { + "epoch": 1.58, + "learning_rate": 2.6326923076923076e-07, + "logits/chosen": -2.5517020225524902, + "logits/rejected": -2.2331695556640625, + "logps/chosen": -306.2018127441406, + "logps/rejected": -289.97216796875, + "loss": 0.0409, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4713718295097351, + "rewards/margins": 6.912310600280762, + "rewards/rejected": -7.3836822509765625, + "step": 1520 + }, + { + "epoch": 1.59, + "learning_rate": 2.6134615384615385e-07, + "logits/chosen": -2.5724055767059326, + "logits/rejected": -2.276639461517334, + "logps/chosen": -289.84307861328125, + "logps/rejected": -308.78851318359375, + "loss": 0.047, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.45345011353492737, + "rewards/margins": 7.111723899841309, + "rewards/rejected": -7.565174102783203, + "step": 1530 + }, + { + "epoch": 1.6, + "learning_rate": 2.5942307692307694e-07, + "logits/chosen": -2.6910042762756348, + "logits/rejected": -2.519693374633789, + "logps/chosen": -274.81695556640625, + "logps/rejected": -293.1619873046875, + "loss": 0.0652, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -0.8246002197265625, + "rewards/margins": 6.030448913574219, + "rewards/rejected": -6.855049133300781, + "step": 1540 + }, + { + "epoch": 1.61, + "learning_rate": 2.5749999999999997e-07, + "logits/chosen": -2.648815631866455, + "logits/rejected": -2.505194902420044, + "logps/chosen": -251.701416015625, + "logps/rejected": -303.21319580078125, + "loss": 0.0427, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.43140754103660583, + "rewards/margins": 6.856492519378662, + "rewards/rejected": -7.287900447845459, + "step": 1550 + }, + { + "epoch": 1.62, + "learning_rate": 2.5557692307692306e-07, + "logits/chosen": -2.5452957153320312, + "logits/rejected": -2.3541762828826904, + "logps/chosen": -237.29531860351562, + "logps/rejected": -246.6190643310547, + "loss": 0.0499, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.9847968816757202, + "rewards/margins": 5.875502109527588, + "rewards/rejected": -6.860299587249756, + "step": 1560 + }, + { + "epoch": 1.63, + "learning_rate": 2.5365384615384614e-07, + "logits/chosen": -2.754185914993286, + "logits/rejected": -2.343951940536499, + "logps/chosen": -326.2292175292969, + "logps/rejected": -300.82843017578125, + "loss": 0.0456, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6876034736633301, + "rewards/margins": 7.04247522354126, + "rewards/rejected": -7.73007869720459, + "step": 1570 + }, + { + "epoch": 1.64, + "learning_rate": 2.5173076923076923e-07, + "logits/chosen": -2.7579290866851807, + "logits/rejected": -2.269740104675293, + "logps/chosen": -305.60357666015625, + "logps/rejected": -292.82257080078125, + "loss": 0.0444, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.6316174268722534, + "rewards/margins": 7.171195983886719, + "rewards/rejected": -7.802813529968262, + "step": 1580 + }, + { + "epoch": 1.65, + "learning_rate": 2.498076923076923e-07, + "logits/chosen": -2.599862575531006, + "logits/rejected": -2.4854395389556885, + "logps/chosen": -308.61041259765625, + "logps/rejected": -350.6466979980469, + "loss": 0.0351, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.18435214459896088, + "rewards/margins": 6.852914333343506, + "rewards/rejected": -7.037265777587891, + "step": 1590 + }, + { + "epoch": 1.66, + "learning_rate": 2.478846153846154e-07, + "logits/chosen": -2.491466522216797, + "logits/rejected": -2.2569782733917236, + "logps/chosen": -288.4465637207031, + "logps/rejected": -276.3277587890625, + "loss": 0.0651, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.8335868716239929, + "rewards/margins": 6.33389949798584, + "rewards/rejected": -7.16748571395874, + "step": 1600 + }, + { + "epoch": 1.66, + "eval_logits/chosen": -2.646865129470825, + "eval_logits/rejected": -2.3390119075775146, + "eval_logps/chosen": -319.6136474609375, + "eval_logps/rejected": -291.59014892578125, + "eval_loss": 0.3962344229221344, + "eval_rewards/accuracies": 0.8015872836112976, + "eval_rewards/chosen": -2.059701681137085, + "eval_rewards/margins": 2.83184552192688, + "eval_rewards/rejected": -4.891546726226807, + "eval_runtime": 138.1446, + "eval_samples_per_second": 14.478, + "eval_steps_per_second": 0.456, + "step": 1600 + }, + { + "epoch": 1.67, + "learning_rate": 2.4596153846153844e-07, + "logits/chosen": -2.6766560077667236, + "logits/rejected": -2.3890440464019775, + "logps/chosen": -320.79901123046875, + "logps/rejected": -327.23553466796875, + "loss": 0.052, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5842215418815613, + "rewards/margins": 6.163203716278076, + "rewards/rejected": -6.747425079345703, + "step": 1610 + }, + { + "epoch": 1.68, + "learning_rate": 2.4403846153846153e-07, + "logits/chosen": -2.553154706954956, + "logits/rejected": -2.430248737335205, + "logps/chosen": -278.9110107421875, + "logps/rejected": -300.0827331542969, + "loss": 0.0549, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.6223709583282471, + "rewards/margins": 6.104709625244141, + "rewards/rejected": -6.727081298828125, + "step": 1620 + }, + { + "epoch": 1.69, + "learning_rate": 2.421153846153846e-07, + "logits/chosen": -2.514636993408203, + "logits/rejected": -2.49489164352417, + "logps/chosen": -264.50677490234375, + "logps/rejected": -279.73504638671875, + "loss": 0.0512, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.4848340153694153, + "rewards/margins": 6.083920478820801, + "rewards/rejected": -6.56875467300415, + "step": 1630 + }, + { + "epoch": 1.7, + "learning_rate": 2.401923076923077e-07, + "logits/chosen": -2.6115622520446777, + "logits/rejected": -2.155635356903076, + "logps/chosen": -317.70391845703125, + "logps/rejected": -309.3824768066406, + "loss": 0.0424, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.23578333854675293, + "rewards/margins": 6.9274091720581055, + "rewards/rejected": -7.1631927490234375, + "step": 1640 + }, + { + "epoch": 1.71, + "learning_rate": 2.3826923076923076e-07, + "logits/chosen": -2.519632577896118, + "logits/rejected": -2.1128008365631104, + "logps/chosen": -304.7782287597656, + "logps/rejected": -318.24578857421875, + "loss": 0.0604, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.653046727180481, + "rewards/margins": 7.262368202209473, + "rewards/rejected": -7.915414333343506, + "step": 1650 + }, + { + "epoch": 1.72, + "learning_rate": 2.3634615384615382e-07, + "logits/chosen": -2.471069812774658, + "logits/rejected": -2.2706027030944824, + "logps/chosen": -321.79833984375, + "logps/rejected": -329.3187255859375, + "loss": 0.0502, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.35187727212905884, + "rewards/margins": 7.088311195373535, + "rewards/rejected": -7.440188407897949, + "step": 1660 + }, + { + "epoch": 1.73, + "learning_rate": 2.344230769230769e-07, + "logits/chosen": -2.7176525592803955, + "logits/rejected": -2.316293478012085, + "logps/chosen": -301.23724365234375, + "logps/rejected": -292.47515869140625, + "loss": 0.0461, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.00272883172146976, + "rewards/margins": 6.684892177581787, + "rewards/rejected": -6.687620639801025, + "step": 1670 + }, + { + "epoch": 1.74, + "learning_rate": 2.325e-07, + "logits/chosen": -2.51845383644104, + "logits/rejected": -2.6053450107574463, + "logps/chosen": -260.30267333984375, + "logps/rejected": -285.4233703613281, + "loss": 0.072, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.8373669385910034, + "rewards/margins": 6.566694736480713, + "rewards/rejected": -7.404061794281006, + "step": 1680 + }, + { + "epoch": 1.75, + "learning_rate": 2.3057692307692306e-07, + "logits/chosen": -2.6345601081848145, + "logits/rejected": -2.0915791988372803, + "logps/chosen": -293.4105529785156, + "logps/rejected": -283.9396057128906, + "loss": 0.0703, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.2911089062690735, + "rewards/margins": 6.6237335205078125, + "rewards/rejected": -6.914842128753662, + "step": 1690 + }, + { + "epoch": 1.77, + "learning_rate": 2.2865384615384615e-07, + "logits/chosen": -2.5609567165374756, + "logits/rejected": -2.4597666263580322, + "logps/chosen": -283.018798828125, + "logps/rejected": -314.49749755859375, + "loss": 0.0738, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.472573459148407, + "rewards/margins": 6.736408233642578, + "rewards/rejected": -7.208981990814209, + "step": 1700 + }, + { + "epoch": 1.77, + "eval_logits/chosen": -2.6606547832489014, + "eval_logits/rejected": -2.353179454803467, + "eval_logps/chosen": -317.9099426269531, + "eval_logps/rejected": -288.78173828125, + "eval_loss": 0.3941672146320343, + "eval_rewards/accuracies": 0.8134920597076416, + "eval_rewards/chosen": -1.889327049255371, + "eval_rewards/margins": 2.721378803253174, + "eval_rewards/rejected": -4.610706329345703, + "eval_runtime": 137.7756, + "eval_samples_per_second": 14.516, + "eval_steps_per_second": 0.457, + "step": 1700 + }, + { + "epoch": 1.78, + "learning_rate": 2.2673076923076923e-07, + "logits/chosen": -2.5994811058044434, + "logits/rejected": -2.188270092010498, + "logps/chosen": -293.59429931640625, + "logps/rejected": -287.1206359863281, + "loss": 0.0583, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -0.12811625003814697, + "rewards/margins": 6.462324619293213, + "rewards/rejected": -6.5904412269592285, + "step": 1710 + }, + { + "epoch": 1.79, + "learning_rate": 2.2480769230769232e-07, + "logits/chosen": -2.701138496398926, + "logits/rejected": -2.504826307296753, + "logps/chosen": -334.2485046386719, + "logps/rejected": -309.32330322265625, + "loss": 0.0506, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.11231260001659393, + "rewards/margins": 6.020442485809326, + "rewards/rejected": -5.908129692077637, + "step": 1720 + }, + { + "epoch": 1.8, + "learning_rate": 2.2288461538461535e-07, + "logits/chosen": -2.6742324829101562, + "logits/rejected": -2.340301275253296, + "logps/chosen": -266.2740478515625, + "logps/rejected": -298.72637939453125, + "loss": 0.0524, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.3445238471031189, + "rewards/margins": 6.339863300323486, + "rewards/rejected": -6.68438720703125, + "step": 1730 + }, + { + "epoch": 1.81, + "learning_rate": 2.2096153846153844e-07, + "logits/chosen": -2.706620693206787, + "logits/rejected": -2.4908840656280518, + "logps/chosen": -304.2544860839844, + "logps/rejected": -328.808349609375, + "loss": 0.0486, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.4945853352546692, + "rewards/margins": 6.773747444152832, + "rewards/rejected": -7.268332481384277, + "step": 1740 + }, + { + "epoch": 1.82, + "learning_rate": 2.1903846153846153e-07, + "logits/chosen": -2.5668416023254395, + "logits/rejected": -2.1745359897613525, + "logps/chosen": -283.23101806640625, + "logps/rejected": -289.1015625, + "loss": 0.1597, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.4894266128540039, + "rewards/margins": 6.333164691925049, + "rewards/rejected": -6.8225908279418945, + "step": 1750 + }, + { + "epoch": 1.83, + "learning_rate": 2.1711538461538462e-07, + "logits/chosen": -2.4643194675445557, + "logits/rejected": -2.2983975410461426, + "logps/chosen": -276.0385437011719, + "logps/rejected": -261.1243591308594, + "loss": 0.0719, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.7288832068443298, + "rewards/margins": 5.606365203857422, + "rewards/rejected": -6.3352484703063965, + "step": 1760 + }, + { + "epoch": 1.84, + "learning_rate": 2.1519230769230768e-07, + "logits/chosen": -2.552398681640625, + "logits/rejected": -2.3484368324279785, + "logps/chosen": -356.1564636230469, + "logps/rejected": -358.2718811035156, + "loss": 0.0665, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 0.34922653436660767, + "rewards/margins": 6.843531608581543, + "rewards/rejected": -6.494305610656738, + "step": 1770 + }, + { + "epoch": 1.85, + "learning_rate": 2.1326923076923076e-07, + "logits/chosen": -2.5096826553344727, + "logits/rejected": -2.1927523612976074, + "logps/chosen": -311.0741882324219, + "logps/rejected": -288.48016357421875, + "loss": 0.0548, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.4201888144016266, + "rewards/margins": 6.081272602081299, + "rewards/rejected": -6.501461029052734, + "step": 1780 + }, + { + "epoch": 1.86, + "learning_rate": 2.1134615384615385e-07, + "logits/chosen": -2.5754733085632324, + "logits/rejected": -2.302943468093872, + "logps/chosen": -319.63763427734375, + "logps/rejected": -306.80706787109375, + "loss": 0.0498, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -0.2358233481645584, + "rewards/margins": 6.327439308166504, + "rewards/rejected": -6.563262939453125, + "step": 1790 + }, + { + "epoch": 1.87, + "learning_rate": 2.094230769230769e-07, + "logits/chosen": -2.5756051540374756, + "logits/rejected": -2.2062323093414307, + "logps/chosen": -303.8487548828125, + "logps/rejected": -291.21343994140625, + "loss": 0.0597, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.1234605461359024, + "rewards/margins": 6.422218322753906, + "rewards/rejected": -6.545678615570068, + "step": 1800 + }, + { + "epoch": 1.87, + "eval_logits/chosen": -2.5907535552978516, + "eval_logits/rejected": -2.2728497982025146, + "eval_logps/chosen": -317.79052734375, + "eval_logps/rejected": -289.8961486816406, + "eval_loss": 0.39897972345352173, + "eval_rewards/accuracies": 0.817460298538208, + "eval_rewards/chosen": -1.8773850202560425, + "eval_rewards/margins": 2.8447630405426025, + "eval_rewards/rejected": -4.7221479415893555, + "eval_runtime": 138.1312, + "eval_samples_per_second": 14.479, + "eval_steps_per_second": 0.456, + "step": 1800 + }, + { + "epoch": 1.88, + "learning_rate": 2.0749999999999997e-07, + "logits/chosen": -2.529545545578003, + "logits/rejected": -2.288567304611206, + "logps/chosen": -281.3018798828125, + "logps/rejected": -325.45086669921875, + "loss": 0.0457, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7181668877601624, + "rewards/margins": 6.582350730895996, + "rewards/rejected": -7.3005170822143555, + "step": 1810 + }, + { + "epoch": 1.89, + "learning_rate": 2.0557692307692306e-07, + "logits/chosen": -2.677251100540161, + "logits/rejected": -2.332127571105957, + "logps/chosen": -365.18426513671875, + "logps/rejected": -319.1354064941406, + "loss": 0.0568, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.353956401348114, + "rewards/margins": 6.727086544036865, + "rewards/rejected": -6.373129844665527, + "step": 1820 + }, + { + "epoch": 1.9, + "learning_rate": 2.0365384615384615e-07, + "logits/chosen": -2.4742937088012695, + "logits/rejected": -2.227391004562378, + "logps/chosen": -290.05865478515625, + "logps/rejected": -287.7528991699219, + "loss": 0.0645, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.4824981689453125, + "rewards/margins": 6.790109157562256, + "rewards/rejected": -7.272607326507568, + "step": 1830 + }, + { + "epoch": 1.91, + "learning_rate": 2.0173076923076923e-07, + "logits/chosen": -2.45625901222229, + "logits/rejected": -2.1974668502807617, + "logps/chosen": -286.39642333984375, + "logps/rejected": -305.8327941894531, + "loss": 0.0741, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -0.8668807744979858, + "rewards/margins": 6.204115867614746, + "rewards/rejected": -7.0709967613220215, + "step": 1840 + }, + { + "epoch": 1.92, + "learning_rate": 1.998076923076923e-07, + "logits/chosen": -2.4767487049102783, + "logits/rejected": -2.2565927505493164, + "logps/chosen": -312.245849609375, + "logps/rejected": -325.4161071777344, + "loss": 0.0461, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.23089423775672913, + "rewards/margins": 6.206521987915039, + "rewards/rejected": -6.437416076660156, + "step": 1850 + }, + { + "epoch": 1.93, + "learning_rate": 1.9788461538461538e-07, + "logits/chosen": -2.4799299240112305, + "logits/rejected": -2.1780128479003906, + "logps/chosen": -337.66900634765625, + "logps/rejected": -293.83551025390625, + "loss": 0.0667, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": -0.772417426109314, + "rewards/margins": 5.717872619628906, + "rewards/rejected": -6.490289211273193, + "step": 1860 + }, + { + "epoch": 1.94, + "learning_rate": 1.9596153846153847e-07, + "logits/chosen": -2.6148133277893066, + "logits/rejected": -2.4374372959136963, + "logps/chosen": -324.8668212890625, + "logps/rejected": -370.79241943359375, + "loss": 0.0593, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.5282601118087769, + "rewards/margins": 6.644644260406494, + "rewards/rejected": -7.172904014587402, + "step": 1870 + }, + { + "epoch": 1.95, + "learning_rate": 1.9403846153846153e-07, + "logits/chosen": -2.567094326019287, + "logits/rejected": -2.3295035362243652, + "logps/chosen": -265.49188232421875, + "logps/rejected": -281.04522705078125, + "loss": 0.0565, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -0.3976692259311676, + "rewards/margins": 5.820679664611816, + "rewards/rejected": -6.218348503112793, + "step": 1880 + }, + { + "epoch": 1.96, + "learning_rate": 1.921153846153846e-07, + "logits/chosen": -2.454172372817993, + "logits/rejected": -2.171295166015625, + "logps/chosen": -271.5619201660156, + "logps/rejected": -311.7131042480469, + "loss": 0.1045, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.039119429886341095, + "rewards/margins": 6.767712593078613, + "rewards/rejected": -6.806832790374756, + "step": 1890 + }, + { + "epoch": 1.97, + "learning_rate": 1.9019230769230768e-07, + "logits/chosen": -2.5482497215270996, + "logits/rejected": -2.401669979095459, + "logps/chosen": -307.3960266113281, + "logps/rejected": -318.3688659667969, + "loss": 0.0686, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 0.022620106115937233, + "rewards/margins": 6.840855598449707, + "rewards/rejected": -6.818235874176025, + "step": 1900 + }, + { + "epoch": 1.97, + "eval_logits/chosen": -2.5658326148986816, + "eval_logits/rejected": -2.255352258682251, + "eval_logps/chosen": -317.76165771484375, + "eval_logps/rejected": -289.4820861816406, + "eval_loss": 0.39241212606430054, + "eval_rewards/accuracies": 0.8055555820465088, + "eval_rewards/chosen": -1.8745005130767822, + "eval_rewards/margins": 2.806236505508423, + "eval_rewards/rejected": -4.680737495422363, + "eval_runtime": 138.2704, + "eval_samples_per_second": 14.464, + "eval_steps_per_second": 0.456, + "step": 1900 + }, + { + "epoch": 1.98, + "learning_rate": 1.8826923076923076e-07, + "logits/chosen": -2.465681552886963, + "logits/rejected": -2.326996326446533, + "logps/chosen": -274.81573486328125, + "logps/rejected": -284.34405517578125, + "loss": 0.0609, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": -0.3031781315803528, + "rewards/margins": 6.008488178253174, + "rewards/rejected": -6.311666488647461, + "step": 1910 + }, + { + "epoch": 1.99, + "learning_rate": 1.8634615384615385e-07, + "logits/chosen": -2.532928943634033, + "logits/rejected": -2.366983652114868, + "logps/chosen": -390.73223876953125, + "logps/rejected": -315.1780700683594, + "loss": 0.0527, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.22381691634655, + "rewards/margins": 6.888078212738037, + "rewards/rejected": -7.1118950843811035, + "step": 1920 + }, + { + "epoch": 2.0, + "learning_rate": 1.844230769230769e-07, + "logits/chosen": -2.545893669128418, + "logits/rejected": -2.114891529083252, + "logps/chosen": -240.4969940185547, + "logps/rejected": -269.58819580078125, + "loss": 0.0416, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 0.3161620497703552, + "rewards/margins": 7.036616325378418, + "rewards/rejected": -6.720454216003418, + "step": 1930 + }, + { + "epoch": 2.01, + "learning_rate": 1.825e-07, + "logits/chosen": -2.4374775886535645, + "logits/rejected": -2.175703525543213, + "logps/chosen": -272.4288024902344, + "logps/rejected": -296.0552673339844, + "loss": 0.0147, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.035207681357860565, + "rewards/margins": 6.330394744873047, + "rewards/rejected": -6.365602493286133, + "step": 1940 + }, + { + "epoch": 2.02, + "learning_rate": 1.8057692307692306e-07, + "logits/chosen": -2.3066885471343994, + "logits/rejected": -2.355970859527588, + "logps/chosen": -294.2997131347656, + "logps/rejected": -333.3860778808594, + "loss": 0.0162, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.04359764978289604, + "rewards/margins": 9.196569442749023, + "rewards/rejected": -9.152971267700195, + "step": 1950 + }, + { + "epoch": 2.04, + "learning_rate": 1.7865384615384615e-07, + "logits/chosen": -2.4848382472991943, + "logits/rejected": -2.32381010055542, + "logps/chosen": -319.75018310546875, + "logps/rejected": -326.4351501464844, + "loss": 0.0142, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.26576751470565796, + "rewards/margins": 6.980968475341797, + "rewards/rejected": -7.2467360496521, + "step": 1960 + }, + { + "epoch": 2.05, + "learning_rate": 1.767307692307692e-07, + "logits/chosen": -2.5374231338500977, + "logits/rejected": -2.332153797149658, + "logps/chosen": -284.25946044921875, + "logps/rejected": -338.75006103515625, + "loss": 0.0138, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31498903036117554, + "rewards/margins": 8.299997329711914, + "rewards/rejected": -7.985008239746094, + "step": 1970 + }, + { + "epoch": 2.06, + "learning_rate": 1.748076923076923e-07, + "logits/chosen": -2.487070083618164, + "logits/rejected": -2.249248504638672, + "logps/chosen": -291.6103820800781, + "logps/rejected": -322.8634948730469, + "loss": 0.0133, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5083962082862854, + "rewards/margins": 7.409769535064697, + "rewards/rejected": -7.918165683746338, + "step": 1980 + }, + { + "epoch": 2.07, + "learning_rate": 1.7288461538461538e-07, + "logits/chosen": -2.5644381046295166, + "logits/rejected": -2.2769272327423096, + "logps/chosen": -264.1942138671875, + "logps/rejected": -300.13299560546875, + "loss": 0.0124, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.47016096115112305, + "rewards/margins": 7.2947282791137695, + "rewards/rejected": -7.764888763427734, + "step": 1990 + }, + { + "epoch": 2.08, + "learning_rate": 1.7096153846153847e-07, + "logits/chosen": -2.40687894821167, + "logits/rejected": -2.2331576347351074, + "logps/chosen": -330.56280517578125, + "logps/rejected": -339.08087158203125, + "loss": 0.0116, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6595343947410583, + "rewards/margins": 7.104054927825928, + "rewards/rejected": -7.763589382171631, + "step": 2000 + }, + { + "epoch": 2.08, + "eval_logits/chosen": -2.534712553024292, + "eval_logits/rejected": -2.2297492027282715, + "eval_logps/chosen": -323.70367431640625, + "eval_logps/rejected": -299.8647155761719, + "eval_loss": 0.426045686006546, + "eval_rewards/accuracies": 0.7936508059501648, + "eval_rewards/chosen": -2.468703508377075, + "eval_rewards/margins": 3.2502999305725098, + "eval_rewards/rejected": -5.719003677368164, + "eval_runtime": 130.9272, + "eval_samples_per_second": 15.276, + "eval_steps_per_second": 0.481, + "step": 2000 + }, + { + "epoch": 2.09, + "learning_rate": 1.6903846153846153e-07, + "logits/chosen": -2.53375506401062, + "logits/rejected": -2.4274983406066895, + "logps/chosen": -317.34210205078125, + "logps/rejected": -329.56561279296875, + "loss": 0.0071, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.4780617356300354, + "rewards/margins": 7.658169746398926, + "rewards/rejected": -8.13623046875, + "step": 2010 + }, + { + "epoch": 2.1, + "learning_rate": 1.671153846153846e-07, + "logits/chosen": -2.3600234985351562, + "logits/rejected": -2.2371509075164795, + "logps/chosen": -238.1803741455078, + "logps/rejected": -311.7017517089844, + "loss": 0.0082, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.047269344329834, + "rewards/margins": 7.484587669372559, + "rewards/rejected": -8.53185749053955, + "step": 2020 + }, + { + "epoch": 2.11, + "learning_rate": 1.6519230769230768e-07, + "logits/chosen": -2.5380847454071045, + "logits/rejected": -2.3048927783966064, + "logps/chosen": -294.823486328125, + "logps/rejected": -327.47802734375, + "loss": 0.014, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.806699275970459, + "rewards/margins": 7.580696105957031, + "rewards/rejected": -8.387394905090332, + "step": 2030 + }, + { + "epoch": 2.12, + "learning_rate": 1.6326923076923077e-07, + "logits/chosen": -2.3959903717041016, + "logits/rejected": -2.3024260997772217, + "logps/chosen": -309.2193298339844, + "logps/rejected": -318.51409912109375, + "loss": 0.0115, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8904142379760742, + "rewards/margins": 7.762335300445557, + "rewards/rejected": -8.652750015258789, + "step": 2040 + }, + { + "epoch": 2.13, + "learning_rate": 1.6134615384615383e-07, + "logits/chosen": -2.4456429481506348, + "logits/rejected": -2.5010082721710205, + "logps/chosen": -293.930908203125, + "logps/rejected": -353.22210693359375, + "loss": 0.009, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8601284027099609, + "rewards/margins": 8.212738990783691, + "rewards/rejected": -9.072866439819336, + "step": 2050 + }, + { + "epoch": 2.14, + "learning_rate": 1.5942307692307691e-07, + "logits/chosen": -2.4511656761169434, + "logits/rejected": -2.115886926651001, + "logps/chosen": -305.025390625, + "logps/rejected": -289.225830078125, + "loss": 0.0092, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.959731936454773, + "rewards/margins": 7.51929235458374, + "rewards/rejected": -8.479024887084961, + "step": 2060 + }, + { + "epoch": 2.15, + "learning_rate": 1.575e-07, + "logits/chosen": -2.4998791217803955, + "logits/rejected": -2.253462314605713, + "logps/chosen": -289.7786560058594, + "logps/rejected": -333.38641357421875, + "loss": 0.0092, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2887849807739258, + "rewards/margins": 7.364945411682129, + "rewards/rejected": -8.653730392456055, + "step": 2070 + }, + { + "epoch": 2.16, + "learning_rate": 1.555769230769231e-07, + "logits/chosen": -2.535858631134033, + "logits/rejected": -2.194636583328247, + "logps/chosen": -300.7867736816406, + "logps/rejected": -307.16937255859375, + "loss": 0.0085, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0742636919021606, + "rewards/margins": 7.198313236236572, + "rewards/rejected": -8.272576332092285, + "step": 2080 + }, + { + "epoch": 2.17, + "learning_rate": 1.5365384615384612e-07, + "logits/chosen": -2.508073329925537, + "logits/rejected": -2.2332749366760254, + "logps/chosen": -277.55584716796875, + "logps/rejected": -303.4755554199219, + "loss": 0.0096, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.042938232421875, + "rewards/margins": 7.390425682067871, + "rewards/rejected": -8.43336296081543, + "step": 2090 + }, + { + "epoch": 2.18, + "learning_rate": 1.517307692307692e-07, + "logits/chosen": -2.4870352745056152, + "logits/rejected": -2.193904161453247, + "logps/chosen": -278.8507385253906, + "logps/rejected": -316.874755859375, + "loss": 0.0114, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -1.6408751010894775, + "rewards/margins": 7.6175031661987305, + "rewards/rejected": -9.258378982543945, + "step": 2100 + }, + { + "epoch": 2.18, + "eval_logits/chosen": -2.521868944168091, + "eval_logits/rejected": -2.218470573425293, + "eval_logps/chosen": -327.2823486328125, + "eval_logps/rejected": -306.3802490234375, + "eval_loss": 0.4518626630306244, + "eval_rewards/accuracies": 0.7976190447807312, + "eval_rewards/chosen": -2.8265669345855713, + "eval_rewards/margins": 3.5439889430999756, + "eval_rewards/rejected": -6.370555400848389, + "eval_runtime": 130.6688, + "eval_samples_per_second": 15.306, + "eval_steps_per_second": 0.482, + "step": 2100 + }, + { + "epoch": 2.19, + "learning_rate": 1.498076923076923e-07, + "logits/chosen": -2.4707891941070557, + "logits/rejected": -2.3319196701049805, + "logps/chosen": -299.81610107421875, + "logps/rejected": -340.24267578125, + "loss": 0.0177, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -1.4307548999786377, + "rewards/margins": 8.416878700256348, + "rewards/rejected": -9.847633361816406, + "step": 2110 + }, + { + "epoch": 2.2, + "learning_rate": 1.4788461538461538e-07, + "logits/chosen": -2.4089057445526123, + "logits/rejected": -2.055510997772217, + "logps/chosen": -286.984375, + "logps/rejected": -311.11590576171875, + "loss": 0.0113, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9583197832107544, + "rewards/margins": 8.543428421020508, + "rewards/rejected": -9.501748085021973, + "step": 2120 + }, + { + "epoch": 2.21, + "learning_rate": 1.4596153846153844e-07, + "logits/chosen": -2.5453710556030273, + "logits/rejected": -2.289592981338501, + "logps/chosen": -301.62762451171875, + "logps/rejected": -309.4609069824219, + "loss": 0.013, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8477941751480103, + "rewards/margins": 7.631629943847656, + "rewards/rejected": -8.479424476623535, + "step": 2130 + }, + { + "epoch": 2.22, + "learning_rate": 1.4403846153846153e-07, + "logits/chosen": -2.5285215377807617, + "logits/rejected": -2.17012095451355, + "logps/chosen": -317.62713623046875, + "logps/rejected": -300.0318298339844, + "loss": 0.0087, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4716318845748901, + "rewards/margins": 7.3103203773498535, + "rewards/rejected": -8.781950950622559, + "step": 2140 + }, + { + "epoch": 2.23, + "learning_rate": 1.4211538461538462e-07, + "logits/chosen": -2.333951473236084, + "logits/rejected": -2.1678061485290527, + "logps/chosen": -298.3710021972656, + "logps/rejected": -300.546630859375, + "loss": 0.0099, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9506980180740356, + "rewards/margins": 7.881108283996582, + "rewards/rejected": -8.831806182861328, + "step": 2150 + }, + { + "epoch": 2.24, + "learning_rate": 1.401923076923077e-07, + "logits/chosen": -2.359285354614258, + "logits/rejected": -2.172787666320801, + "logps/chosen": -273.92962646484375, + "logps/rejected": -312.80499267578125, + "loss": 0.0092, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1501799821853638, + "rewards/margins": 7.2449822425842285, + "rewards/rejected": -8.395162582397461, + "step": 2160 + }, + { + "epoch": 2.25, + "learning_rate": 1.3826923076923074e-07, + "logits/chosen": -2.4522316455841064, + "logits/rejected": -2.1035475730895996, + "logps/chosen": -273.8643493652344, + "logps/rejected": -292.98089599609375, + "loss": 0.0103, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4371745586395264, + "rewards/margins": 8.328310012817383, + "rewards/rejected": -9.765485763549805, + "step": 2170 + }, + { + "epoch": 2.26, + "learning_rate": 1.3634615384615383e-07, + "logits/chosen": -2.516073703765869, + "logits/rejected": -2.171781539916992, + "logps/chosen": -346.954833984375, + "logps/rejected": -358.9801940917969, + "loss": 0.0099, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9951909780502319, + "rewards/margins": 8.938260078430176, + "rewards/rejected": -9.933451652526855, + "step": 2180 + }, + { + "epoch": 2.27, + "learning_rate": 1.3442307692307691e-07, + "logits/chosen": -2.3622655868530273, + "logits/rejected": -2.100069761276245, + "logps/chosen": -321.36663818359375, + "logps/rejected": -338.2700500488281, + "loss": 0.0097, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5687800049781799, + "rewards/margins": 8.284536361694336, + "rewards/rejected": -8.853315353393555, + "step": 2190 + }, + { + "epoch": 2.28, + "learning_rate": 1.325e-07, + "logits/chosen": -2.537001848220825, + "logits/rejected": -2.2472758293151855, + "logps/chosen": -326.72735595703125, + "logps/rejected": -396.21026611328125, + "loss": 0.0073, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1825602054595947, + "rewards/margins": 8.144112586975098, + "rewards/rejected": -9.326672554016113, + "step": 2200 + }, + { + "epoch": 2.28, + "eval_logits/chosen": -2.512586832046509, + "eval_logits/rejected": -2.2102549076080322, + "eval_logps/chosen": -328.43841552734375, + "eval_logps/rejected": -308.2383728027344, + "eval_loss": 0.4562525749206543, + "eval_rewards/accuracies": 0.8015872836112976, + "eval_rewards/chosen": -2.9421753883361816, + "eval_rewards/margins": 3.614194869995117, + "eval_rewards/rejected": -6.556369781494141, + "eval_runtime": 130.4834, + "eval_samples_per_second": 15.328, + "eval_steps_per_second": 0.483, + "step": 2200 + }, + { + "epoch": 2.29, + "learning_rate": 1.305769230769231e-07, + "logits/chosen": -2.5175909996032715, + "logits/rejected": -2.245337963104248, + "logps/chosen": -322.72509765625, + "logps/rejected": -354.58154296875, + "loss": 0.0079, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.8563722372055054, + "rewards/margins": 8.002213478088379, + "rewards/rejected": -8.858585357666016, + "step": 2210 + }, + { + "epoch": 2.31, + "learning_rate": 1.2865384615384615e-07, + "logits/chosen": -2.554388999938965, + "logits/rejected": -2.2140610218048096, + "logps/chosen": -298.94482421875, + "logps/rejected": -368.2962341308594, + "loss": 0.0096, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.3428957164287567, + "rewards/margins": 8.27833080291748, + "rewards/rejected": -8.62122631072998, + "step": 2220 + }, + { + "epoch": 2.32, + "learning_rate": 1.2673076923076924e-07, + "logits/chosen": -2.54005765914917, + "logits/rejected": -2.136395215988159, + "logps/chosen": -334.72747802734375, + "logps/rejected": -328.95953369140625, + "loss": 0.0083, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.920019805431366, + "rewards/margins": 8.396950721740723, + "rewards/rejected": -9.316969871520996, + "step": 2230 + }, + { + "epoch": 2.33, + "learning_rate": 1.248076923076923e-07, + "logits/chosen": -2.515563726425171, + "logits/rejected": -2.100313425064087, + "logps/chosen": -304.0508117675781, + "logps/rejected": -331.5612487792969, + "loss": 0.0085, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.7162388563156128, + "rewards/margins": 8.973624229431152, + "rewards/rejected": -9.689862251281738, + "step": 2240 + }, + { + "epoch": 2.34, + "learning_rate": 1.2288461538461538e-07, + "logits/chosen": -2.5252881050109863, + "logits/rejected": -2.220909595489502, + "logps/chosen": -353.8068542480469, + "logps/rejected": -327.639892578125, + "loss": 0.008, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.5056819915771484, + "rewards/margins": 8.935028076171875, + "rewards/rejected": -9.440710067749023, + "step": 2250 + }, + { + "epoch": 2.35, + "learning_rate": 1.2096153846153847e-07, + "logits/chosen": -2.464665412902832, + "logits/rejected": -2.2449238300323486, + "logps/chosen": -337.83270263671875, + "logps/rejected": -318.53338623046875, + "loss": 0.0076, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0805784463882446, + "rewards/margins": 7.592961311340332, + "rewards/rejected": -8.673539161682129, + "step": 2260 + }, + { + "epoch": 2.36, + "learning_rate": 1.1903846153846153e-07, + "logits/chosen": -2.430225133895874, + "logits/rejected": -2.233182191848755, + "logps/chosen": -288.67864990234375, + "logps/rejected": -340.03253173828125, + "loss": 0.0066, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9145647883415222, + "rewards/margins": 8.35079288482666, + "rewards/rejected": -9.26535701751709, + "step": 2270 + }, + { + "epoch": 2.37, + "learning_rate": 1.1711538461538462e-07, + "logits/chosen": -2.484591245651245, + "logits/rejected": -2.2712762355804443, + "logps/chosen": -304.85809326171875, + "logps/rejected": -307.70098876953125, + "loss": 0.0062, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0181230306625366, + "rewards/margins": 8.104196548461914, + "rewards/rejected": -9.122319221496582, + "step": 2280 + }, + { + "epoch": 2.38, + "learning_rate": 1.1519230769230768e-07, + "logits/chosen": -2.600891590118408, + "logits/rejected": -2.2220699787139893, + "logps/chosen": -368.4697570800781, + "logps/rejected": -334.89471435546875, + "loss": 0.0096, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.5690144300460815, + "rewards/margins": 8.74263858795166, + "rewards/rejected": -9.311653137207031, + "step": 2290 + }, + { + "epoch": 2.39, + "learning_rate": 1.1326923076923077e-07, + "logits/chosen": -2.51472544670105, + "logits/rejected": -2.2086689472198486, + "logps/chosen": -311.0439453125, + "logps/rejected": -318.931396484375, + "loss": 0.0094, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5231196880340576, + "rewards/margins": 8.030439376831055, + "rewards/rejected": -9.553558349609375, + "step": 2300 + }, + { + "epoch": 2.39, + "eval_logits/chosen": -2.5081491470336914, + "eval_logits/rejected": -2.205916404724121, + "eval_logps/chosen": -332.2627868652344, + "eval_logps/rejected": -313.21649169921875, + "eval_loss": 0.4636384844779968, + "eval_rewards/accuracies": 0.8015872836112976, + "eval_rewards/chosen": -3.3246116638183594, + "eval_rewards/margins": 3.7295706272125244, + "eval_rewards/rejected": -7.054182052612305, + "eval_runtime": 130.8912, + "eval_samples_per_second": 15.28, + "eval_steps_per_second": 0.481, + "step": 2300 + }, + { + "epoch": 2.4, + "learning_rate": 1.1134615384615384e-07, + "logits/chosen": -2.4573376178741455, + "logits/rejected": -2.0740745067596436, + "logps/chosen": -303.64959716796875, + "logps/rejected": -306.9479064941406, + "loss": 0.0051, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2850396633148193, + "rewards/margins": 8.17421817779541, + "rewards/rejected": -9.459257125854492, + "step": 2310 + }, + { + "epoch": 2.41, + "learning_rate": 1.0942307692307693e-07, + "logits/chosen": -2.4275412559509277, + "logits/rejected": -2.2551393508911133, + "logps/chosen": -307.6458740234375, + "logps/rejected": -377.93280029296875, + "loss": 0.0057, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.249359369277954, + "rewards/margins": 8.990339279174805, + "rewards/rejected": -10.23969841003418, + "step": 2320 + }, + { + "epoch": 2.42, + "learning_rate": 1.0749999999999999e-07, + "logits/chosen": -2.548478126525879, + "logits/rejected": -2.3526339530944824, + "logps/chosen": -295.4488220214844, + "logps/rejected": -376.52154541015625, + "loss": 0.008, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1120712757110596, + "rewards/margins": 8.262701988220215, + "rewards/rejected": -9.374773979187012, + "step": 2330 + }, + { + "epoch": 2.43, + "learning_rate": 1.0557692307692308e-07, + "logits/chosen": -2.5474133491516113, + "logits/rejected": -2.391606092453003, + "logps/chosen": -334.513671875, + "logps/rejected": -345.8397521972656, + "loss": 0.0071, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.15473210811615, + "rewards/margins": 8.487726211547852, + "rewards/rejected": -9.64245891571045, + "step": 2340 + }, + { + "epoch": 2.44, + "learning_rate": 1.0365384615384615e-07, + "logits/chosen": -2.438559055328369, + "logits/rejected": -2.14943528175354, + "logps/chosen": -308.0134582519531, + "logps/rejected": -321.0350341796875, + "loss": 0.0066, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2040348052978516, + "rewards/margins": 8.601288795471191, + "rewards/rejected": -9.805322647094727, + "step": 2350 + }, + { + "epoch": 2.45, + "learning_rate": 1.0173076923076922e-07, + "logits/chosen": -2.4214491844177246, + "logits/rejected": -2.230001211166382, + "logps/chosen": -316.44677734375, + "logps/rejected": -337.9944763183594, + "loss": 0.0086, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -0.9195982813835144, + "rewards/margins": 8.974394798278809, + "rewards/rejected": -9.89399242401123, + "step": 2360 + }, + { + "epoch": 2.46, + "learning_rate": 9.98076923076923e-08, + "logits/chosen": -2.442124605178833, + "logits/rejected": -2.130152702331543, + "logps/chosen": -352.26678466796875, + "logps/rejected": -360.30853271484375, + "loss": 0.0071, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2404325008392334, + "rewards/margins": 8.4083890914917, + "rewards/rejected": -9.648821830749512, + "step": 2370 + }, + { + "epoch": 2.47, + "learning_rate": 9.788461538461539e-08, + "logits/chosen": -2.4774975776672363, + "logits/rejected": -2.3399832248687744, + "logps/chosen": -281.23297119140625, + "logps/rejected": -328.12054443359375, + "loss": 0.0094, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.779415488243103, + "rewards/margins": 8.644756317138672, + "rewards/rejected": -10.424173355102539, + "step": 2380 + }, + { + "epoch": 2.48, + "learning_rate": 9.596153846153846e-08, + "logits/chosen": -2.320403575897217, + "logits/rejected": -2.106785297393799, + "logps/chosen": -303.82781982421875, + "logps/rejected": -352.65887451171875, + "loss": 0.0138, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -1.5107743740081787, + "rewards/margins": 9.153158187866211, + "rewards/rejected": -10.663931846618652, + "step": 2390 + }, + { + "epoch": 2.49, + "learning_rate": 9.403846153846153e-08, + "logits/chosen": -2.466916561126709, + "logits/rejected": -2.1457412242889404, + "logps/chosen": -302.87255859375, + "logps/rejected": -329.4913024902344, + "loss": 0.0056, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.343062400817871, + "rewards/margins": 8.397542953491211, + "rewards/rejected": -9.74060344696045, + "step": 2400 + }, + { + "epoch": 2.49, + "eval_logits/chosen": -2.4943490028381348, + "eval_logits/rejected": -2.194486379623413, + "eval_logps/chosen": -332.6160583496094, + "eval_logps/rejected": -314.3266296386719, + "eval_loss": 0.4745297133922577, + "eval_rewards/accuracies": 0.7976190447807312, + "eval_rewards/chosen": -3.3599419593811035, + "eval_rewards/margins": 3.805250883102417, + "eval_rewards/rejected": -7.1651930809021, + "eval_runtime": 131.3732, + "eval_samples_per_second": 15.224, + "eval_steps_per_second": 0.48, + "step": 2400 + }, + { + "epoch": 2.5, + "learning_rate": 9.211538461538461e-08, + "logits/chosen": -2.34041690826416, + "logits/rejected": -2.142360210418701, + "logps/chosen": -270.6195373535156, + "logps/rejected": -337.4690856933594, + "loss": 0.0083, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4843101501464844, + "rewards/margins": 8.575994491577148, + "rewards/rejected": -10.060304641723633, + "step": 2410 + }, + { + "epoch": 2.51, + "learning_rate": 9.01923076923077e-08, + "logits/chosen": -2.4574790000915527, + "logits/rejected": -2.0679962635040283, + "logps/chosen": -332.03607177734375, + "logps/rejected": -336.3428955078125, + "loss": 0.0071, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.087881565093994, + "rewards/margins": 8.020088195800781, + "rewards/rejected": -10.107969284057617, + "step": 2420 + }, + { + "epoch": 2.52, + "learning_rate": 8.826923076923077e-08, + "logits/chosen": -2.3383586406707764, + "logits/rejected": -2.243995189666748, + "logps/chosen": -283.6031188964844, + "logps/rejected": -319.84686279296875, + "loss": 0.0086, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3011380434036255, + "rewards/margins": 8.05530834197998, + "rewards/rejected": -9.3564453125, + "step": 2430 + }, + { + "epoch": 2.53, + "learning_rate": 8.634615384615384e-08, + "logits/chosen": -2.3765769004821777, + "logits/rejected": -2.2421634197235107, + "logps/chosen": -298.99407958984375, + "logps/rejected": -322.83148193359375, + "loss": 0.0056, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1525708436965942, + "rewards/margins": 8.4398832321167, + "rewards/rejected": -9.592453956604004, + "step": 2440 + }, + { + "epoch": 2.54, + "learning_rate": 8.442307692307692e-08, + "logits/chosen": -2.4655745029449463, + "logits/rejected": -2.1042861938476562, + "logps/chosen": -335.84503173828125, + "logps/rejected": -348.6551818847656, + "loss": 0.0122, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3728059530258179, + "rewards/margins": 8.628170013427734, + "rewards/rejected": -10.000975608825684, + "step": 2450 + }, + { + "epoch": 2.55, + "learning_rate": 8.25e-08, + "logits/chosen": -2.4270904064178467, + "logits/rejected": -2.292043447494507, + "logps/chosen": -336.81292724609375, + "logps/rejected": -339.5079345703125, + "loss": 0.0074, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4841727018356323, + "rewards/margins": 8.56916332244873, + "rewards/rejected": -10.053335189819336, + "step": 2460 + }, + { + "epoch": 2.56, + "learning_rate": 8.057692307692306e-08, + "logits/chosen": -2.425565004348755, + "logits/rejected": -2.2447338104248047, + "logps/chosen": -312.3524475097656, + "logps/rejected": -347.9653015136719, + "loss": 0.0059, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.970663070678711, + "rewards/margins": 9.28294849395752, + "rewards/rejected": -11.25361156463623, + "step": 2470 + }, + { + "epoch": 2.58, + "learning_rate": 7.865384615384615e-08, + "logits/chosen": -2.592557191848755, + "logits/rejected": -2.1757986545562744, + "logps/chosen": -331.1345520019531, + "logps/rejected": -357.92486572265625, + "loss": 0.0068, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4531742334365845, + "rewards/margins": 8.858552932739258, + "rewards/rejected": -10.311727523803711, + "step": 2480 + }, + { + "epoch": 2.59, + "learning_rate": 7.673076923076923e-08, + "logits/chosen": -2.4398257732391357, + "logits/rejected": -2.263611316680908, + "logps/chosen": -326.148193359375, + "logps/rejected": -340.4607238769531, + "loss": 0.0063, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3447742462158203, + "rewards/margins": 8.881135940551758, + "rewards/rejected": -10.225909233093262, + "step": 2490 + }, + { + "epoch": 2.6, + "learning_rate": 7.480769230769231e-08, + "logits/chosen": -2.3727784156799316, + "logits/rejected": -2.0641674995422363, + "logps/chosen": -314.0769348144531, + "logps/rejected": -375.218017578125, + "loss": 0.0052, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.9205458164215088, + "rewards/margins": 8.602018356323242, + "rewards/rejected": -10.522564888000488, + "step": 2500 + }, + { + "epoch": 2.6, + "eval_logits/chosen": -2.4880807399749756, + "eval_logits/rejected": -2.188831090927124, + "eval_logps/chosen": -333.9322204589844, + "eval_logps/rejected": -316.0655517578125, + "eval_loss": 0.48117944598197937, + "eval_rewards/accuracies": 0.7976190447807312, + "eval_rewards/chosen": -3.4915573596954346, + "eval_rewards/margins": 3.8475303649902344, + "eval_rewards/rejected": -7.339086532592773, + "eval_runtime": 130.8113, + "eval_samples_per_second": 15.289, + "eval_steps_per_second": 0.482, + "step": 2500 + }, + { + "epoch": 2.61, + "learning_rate": 7.288461538461537e-08, + "logits/chosen": -2.321307897567749, + "logits/rejected": -2.3813178539276123, + "logps/chosen": -296.61102294921875, + "logps/rejected": -338.35260009765625, + "loss": 0.0065, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7034938335418701, + "rewards/margins": 8.526666641235352, + "rewards/rejected": -10.2301607131958, + "step": 2510 + }, + { + "epoch": 2.62, + "learning_rate": 7.096153846153846e-08, + "logits/chosen": -2.401418685913086, + "logits/rejected": -1.986132264137268, + "logps/chosen": -320.1055603027344, + "logps/rejected": -354.8368225097656, + "loss": 0.0155, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -1.6911952495574951, + "rewards/margins": 8.266457557678223, + "rewards/rejected": -9.957651138305664, + "step": 2520 + }, + { + "epoch": 2.63, + "learning_rate": 6.903846153846154e-08, + "logits/chosen": -2.4205477237701416, + "logits/rejected": -2.201981782913208, + "logps/chosen": -280.1654052734375, + "logps/rejected": -321.9850158691406, + "loss": 0.0087, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0640054941177368, + "rewards/margins": 8.131551742553711, + "rewards/rejected": -9.195556640625, + "step": 2530 + }, + { + "epoch": 2.64, + "learning_rate": 6.711538461538461e-08, + "logits/chosen": -2.4457507133483887, + "logits/rejected": -2.2051494121551514, + "logps/chosen": -301.8743591308594, + "logps/rejected": -312.6191711425781, + "loss": 0.0099, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5744168758392334, + "rewards/margins": 7.840787410736084, + "rewards/rejected": -9.415205001831055, + "step": 2540 + }, + { + "epoch": 2.65, + "learning_rate": 6.519230769230768e-08, + "logits/chosen": -2.3616795539855957, + "logits/rejected": -2.280289888381958, + "logps/chosen": -294.35308837890625, + "logps/rejected": -342.05218505859375, + "loss": 0.0083, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.15314781665802, + "rewards/margins": 8.7235107421875, + "rewards/rejected": -9.87665843963623, + "step": 2550 + }, + { + "epoch": 2.66, + "learning_rate": 6.326923076923077e-08, + "logits/chosen": -2.3353302478790283, + "logits/rejected": -2.244074821472168, + "logps/chosen": -258.412353515625, + "logps/rejected": -297.08795166015625, + "loss": 0.0104, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2376277446746826, + "rewards/margins": 8.651219367980957, + "rewards/rejected": -9.888847351074219, + "step": 2560 + }, + { + "epoch": 2.67, + "learning_rate": 6.134615384615384e-08, + "logits/chosen": -2.478097438812256, + "logits/rejected": -2.2386653423309326, + "logps/chosen": -348.7337341308594, + "logps/rejected": -403.3638610839844, + "loss": 0.0058, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3783025741577148, + "rewards/margins": 9.295442581176758, + "rewards/rejected": -10.673746109008789, + "step": 2570 + }, + { + "epoch": 2.68, + "learning_rate": 5.9423076923076925e-08, + "logits/chosen": -2.4538445472717285, + "logits/rejected": -2.183901071548462, + "logps/chosen": -297.26519775390625, + "logps/rejected": -336.0912170410156, + "loss": 0.0094, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6643871068954468, + "rewards/margins": 8.495319366455078, + "rewards/rejected": -10.159708023071289, + "step": 2580 + }, + { + "epoch": 2.69, + "learning_rate": 5.75e-08, + "logits/chosen": -2.4051222801208496, + "logits/rejected": -2.143523931503296, + "logps/chosen": -306.6567077636719, + "logps/rejected": -343.2184143066406, + "loss": 0.0575, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2398414611816406, + "rewards/margins": 9.57859992980957, + "rewards/rejected": -10.818441390991211, + "step": 2590 + }, + { + "epoch": 2.7, + "learning_rate": 5.557692307692308e-08, + "logits/chosen": -2.491457462310791, + "logits/rejected": -2.118168354034424, + "logps/chosen": -327.12127685546875, + "logps/rejected": -316.92608642578125, + "loss": 0.0065, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2390878200531006, + "rewards/margins": 8.092336654663086, + "rewards/rejected": -9.331425666809082, + "step": 2600 + }, + { + "epoch": 2.7, + "eval_logits/chosen": -2.455951690673828, + "eval_logits/rejected": -2.164350748062134, + "eval_logps/chosen": -331.2425231933594, + "eval_logps/rejected": -312.561279296875, + "eval_loss": 0.4678390920162201, + "eval_rewards/accuracies": 0.7976190447807312, + "eval_rewards/chosen": -3.2225840091705322, + "eval_rewards/margins": 3.7660768032073975, + "eval_rewards/rejected": -6.9886603355407715, + "eval_runtime": 130.8661, + "eval_samples_per_second": 15.283, + "eval_steps_per_second": 0.481, + "step": 2600 + }, + { + "epoch": 2.71, + "learning_rate": 5.365384615384615e-08, + "logits/chosen": -2.4199442863464355, + "logits/rejected": -2.348403215408325, + "logps/chosen": -288.18865966796875, + "logps/rejected": -321.6029968261719, + "loss": 0.0074, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4005610942840576, + "rewards/margins": 8.314815521240234, + "rewards/rejected": -9.715375900268555, + "step": 2610 + }, + { + "epoch": 2.72, + "learning_rate": 5.1730769230769234e-08, + "logits/chosen": -2.3591227531433105, + "logits/rejected": -2.2464001178741455, + "logps/chosen": -338.2710876464844, + "logps/rejected": -352.8677673339844, + "loss": 0.0067, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3274883031845093, + "rewards/margins": 8.12449836730957, + "rewards/rejected": -9.451985359191895, + "step": 2620 + }, + { + "epoch": 2.73, + "learning_rate": 4.980769230769231e-08, + "logits/chosen": -2.3736279010772705, + "logits/rejected": -2.245208978652954, + "logps/chosen": -286.21038818359375, + "logps/rejected": -339.1667175292969, + "loss": 0.0074, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.6096528768539429, + "rewards/margins": 8.592547416687012, + "rewards/rejected": -9.202199935913086, + "step": 2630 + }, + { + "epoch": 2.74, + "learning_rate": 4.788461538461538e-08, + "logits/chosen": -2.3171088695526123, + "logits/rejected": -2.2612321376800537, + "logps/chosen": -266.6427001953125, + "logps/rejected": -303.3582458496094, + "loss": 0.0074, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.0699584484100342, + "rewards/margins": 8.39510440826416, + "rewards/rejected": -9.465063095092773, + "step": 2640 + }, + { + "epoch": 2.75, + "learning_rate": 4.596153846153846e-08, + "logits/chosen": -2.322035551071167, + "logits/rejected": -2.1794607639312744, + "logps/chosen": -310.55230712890625, + "logps/rejected": -354.242431640625, + "loss": 0.0065, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9702370762825012, + "rewards/margins": 8.775087356567383, + "rewards/rejected": -9.745325088500977, + "step": 2650 + }, + { + "epoch": 2.76, + "learning_rate": 4.4038461538461536e-08, + "logits/chosen": -2.3565049171447754, + "logits/rejected": -2.1641526222229004, + "logps/chosen": -284.02880859375, + "logps/rejected": -341.21258544921875, + "loss": 0.0057, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5019583702087402, + "rewards/margins": 8.348108291625977, + "rewards/rejected": -9.850065231323242, + "step": 2660 + }, + { + "epoch": 2.77, + "learning_rate": 4.211538461538462e-08, + "logits/chosen": -2.457857608795166, + "logits/rejected": -2.1471848487854004, + "logps/chosen": -322.2108459472656, + "logps/rejected": -342.0418395996094, + "loss": 0.0066, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7656872272491455, + "rewards/margins": 8.020166397094727, + "rewards/rejected": -9.785853385925293, + "step": 2670 + }, + { + "epoch": 2.78, + "learning_rate": 4.019230769230769e-08, + "logits/chosen": -2.314117670059204, + "logits/rejected": -2.1793768405914307, + "logps/chosen": -333.1905822753906, + "logps/rejected": -368.11749267578125, + "loss": 0.0076, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2259396314620972, + "rewards/margins": 8.503653526306152, + "rewards/rejected": -9.729593276977539, + "step": 2680 + }, + { + "epoch": 2.79, + "learning_rate": 3.826923076923077e-08, + "logits/chosen": -2.349207878112793, + "logits/rejected": -2.1943373680114746, + "logps/chosen": -265.8515625, + "logps/rejected": -290.029541015625, + "loss": 0.0067, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.5206115245819092, + "rewards/margins": 8.009859085083008, + "rewards/rejected": -9.530470848083496, + "step": 2690 + }, + { + "epoch": 2.8, + "learning_rate": 3.6346153846153845e-08, + "logits/chosen": -2.2535085678100586, + "logits/rejected": -2.2148807048797607, + "logps/chosen": -296.49786376953125, + "logps/rejected": -332.1768798828125, + "loss": 0.0059, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.0893945693969727, + "rewards/margins": 8.603468894958496, + "rewards/rejected": -10.692864418029785, + "step": 2700 + }, + { + "epoch": 2.8, + "eval_logits/chosen": -2.4482967853546143, + "eval_logits/rejected": -2.157228708267212, + "eval_logps/chosen": -333.32342529296875, + "eval_logps/rejected": -315.1584167480469, + "eval_loss": 0.4693792760372162, + "eval_rewards/accuracies": 0.7976190447807312, + "eval_rewards/chosen": -3.4306719303131104, + "eval_rewards/margins": 3.8177053928375244, + "eval_rewards/rejected": -7.248377323150635, + "eval_runtime": 130.9189, + "eval_samples_per_second": 15.277, + "eval_steps_per_second": 0.481, + "step": 2700 + }, + { + "epoch": 2.81, + "learning_rate": 3.4423076923076926e-08, + "logits/chosen": -2.188178300857544, + "logits/rejected": -2.098421335220337, + "logps/chosen": -260.811279296875, + "logps/rejected": -316.5066223144531, + "loss": 0.01, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3799587488174438, + "rewards/margins": 8.445514678955078, + "rewards/rejected": -9.825471878051758, + "step": 2710 + }, + { + "epoch": 2.82, + "learning_rate": 3.25e-08, + "logits/chosen": -2.4772255420684814, + "logits/rejected": -2.2826056480407715, + "logps/chosen": -332.14886474609375, + "logps/rejected": -351.8216857910156, + "loss": 0.0083, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2810627222061157, + "rewards/margins": 8.637121200561523, + "rewards/rejected": -9.918184280395508, + "step": 2720 + }, + { + "epoch": 2.83, + "learning_rate": 3.0576923076923074e-08, + "logits/chosen": -2.433171510696411, + "logits/rejected": -2.2717080116271973, + "logps/chosen": -310.1302795410156, + "logps/rejected": -352.16851806640625, + "loss": 0.0087, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": -1.9453051090240479, + "rewards/margins": 8.546197891235352, + "rewards/rejected": -10.491501808166504, + "step": 2730 + }, + { + "epoch": 2.85, + "learning_rate": 2.8653846153846154e-08, + "logits/chosen": -2.4593279361724854, + "logits/rejected": -2.2902705669403076, + "logps/chosen": -298.08258056640625, + "logps/rejected": -338.76300048828125, + "loss": 0.0072, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.3809144496917725, + "rewards/margins": 8.587820053100586, + "rewards/rejected": -9.968733787536621, + "step": 2740 + }, + { + "epoch": 2.86, + "learning_rate": 2.673076923076923e-08, + "logits/chosen": -2.4502007961273193, + "logits/rejected": -2.133526563644409, + "logps/chosen": -303.839111328125, + "logps/rejected": -360.4519958496094, + "loss": 0.0073, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.9425987005233765, + "rewards/margins": 8.856470108032227, + "rewards/rejected": -9.799068450927734, + "step": 2750 + }, + { + "epoch": 2.87, + "learning_rate": 2.4807692307692306e-08, + "logits/chosen": -2.3859500885009766, + "logits/rejected": -2.0461232662200928, + "logps/chosen": -288.9315490722656, + "logps/rejected": -340.39483642578125, + "loss": 0.0131, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.621050238609314, + "rewards/margins": 8.439092636108398, + "rewards/rejected": -10.060144424438477, + "step": 2760 + }, + { + "epoch": 2.88, + "learning_rate": 2.2884615384615383e-08, + "logits/chosen": -2.399702787399292, + "logits/rejected": -2.210979700088501, + "logps/chosen": -303.9853515625, + "logps/rejected": -327.6991271972656, + "loss": 0.0092, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2347537279129028, + "rewards/margins": 8.255434036254883, + "rewards/rejected": -9.490187644958496, + "step": 2770 + }, + { + "epoch": 2.89, + "learning_rate": 2.096153846153846e-08, + "logits/chosen": -2.4862875938415527, + "logits/rejected": -1.9970000982284546, + "logps/chosen": -348.6851806640625, + "logps/rejected": -346.09869384765625, + "loss": 0.0069, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.210662841796875, + "rewards/margins": 8.414986610412598, + "rewards/rejected": -9.625649452209473, + "step": 2780 + }, + { + "epoch": 2.9, + "learning_rate": 1.9038461538461537e-08, + "logits/chosen": -2.2640528678894043, + "logits/rejected": -2.353923797607422, + "logps/chosen": -350.0566711425781, + "logps/rejected": -368.6754150390625, + "loss": 0.0065, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4119244813919067, + "rewards/margins": 8.232645034790039, + "rewards/rejected": -9.644569396972656, + "step": 2790 + }, + { + "epoch": 2.91, + "learning_rate": 1.7115384615384615e-08, + "logits/chosen": -2.500624895095825, + "logits/rejected": -2.284454107284546, + "logps/chosen": -296.8025817871094, + "logps/rejected": -343.0960998535156, + "loss": 0.0054, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.786973237991333, + "rewards/margins": 8.618185043334961, + "rewards/rejected": -9.405158996582031, + "step": 2800 + }, + { + "epoch": 2.91, + "eval_logits/chosen": -2.449103832244873, + "eval_logits/rejected": -2.1575379371643066, + "eval_logps/chosen": -333.975830078125, + "eval_logps/rejected": -315.9576416015625, + "eval_loss": 0.47073113918304443, + "eval_rewards/accuracies": 0.8055555820465088, + "eval_rewards/chosen": -3.4959194660186768, + "eval_rewards/margins": 3.832378625869751, + "eval_rewards/rejected": -7.3282976150512695, + "eval_runtime": 131.1669, + "eval_samples_per_second": 15.248, + "eval_steps_per_second": 0.48, + "step": 2800 + }, + { + "epoch": 2.92, + "learning_rate": 1.5192307692307692e-08, + "logits/chosen": -2.2975189685821533, + "logits/rejected": -2.093742847442627, + "logps/chosen": -296.9509582519531, + "logps/rejected": -362.02008056640625, + "loss": 0.0061, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.4728249311447144, + "rewards/margins": 9.322419166564941, + "rewards/rejected": -10.795244216918945, + "step": 2810 + }, + { + "epoch": 2.93, + "learning_rate": 1.326923076923077e-08, + "logits/chosen": -2.4062585830688477, + "logits/rejected": -2.2824714183807373, + "logps/chosen": -352.0885314941406, + "logps/rejected": -359.2705383300781, + "loss": 0.0046, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.8912947177886963, + "rewards/margins": 8.833779335021973, + "rewards/rejected": -10.725072860717773, + "step": 2820 + }, + { + "epoch": 2.94, + "learning_rate": 1.1346153846153845e-08, + "logits/chosen": -2.436295509338379, + "logits/rejected": -2.212411880493164, + "logps/chosen": -306.6794738769531, + "logps/rejected": -373.9212341308594, + "loss": 0.008, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6317508220672607, + "rewards/margins": 9.109142303466797, + "rewards/rejected": -10.74089241027832, + "step": 2830 + }, + { + "epoch": 2.95, + "learning_rate": 9.423076923076922e-09, + "logits/chosen": -2.5206334590911865, + "logits/rejected": -2.230746030807495, + "logps/chosen": -374.86334228515625, + "logps/rejected": -365.6614685058594, + "loss": 0.0068, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.1036624908447266, + "rewards/margins": 9.229228019714355, + "rewards/rejected": -10.332891464233398, + "step": 2840 + }, + { + "epoch": 2.96, + "learning_rate": 7.5e-09, + "logits/chosen": -2.4916155338287354, + "logits/rejected": -2.3132541179656982, + "logps/chosen": -284.1214599609375, + "logps/rejected": -332.753662109375, + "loss": 0.0074, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7909221649169922, + "rewards/margins": 7.715114593505859, + "rewards/rejected": -9.506036758422852, + "step": 2850 + }, + { + "epoch": 2.97, + "learning_rate": 5.576923076923077e-09, + "logits/chosen": -2.560082197189331, + "logits/rejected": -2.2312729358673096, + "logps/chosen": -309.1767272949219, + "logps/rejected": -349.9689025878906, + "loss": 0.0053, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.8000606298446655, + "rewards/margins": 8.335681915283203, + "rewards/rejected": -10.135743141174316, + "step": 2860 + }, + { + "epoch": 2.98, + "learning_rate": 3.6538461538461534e-09, + "logits/chosen": -2.233848810195923, + "logits/rejected": -2.199557304382324, + "logps/chosen": -336.6528015136719, + "logps/rejected": -391.2978820800781, + "loss": 0.0091, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.6596148014068604, + "rewards/margins": 8.436461448669434, + "rewards/rejected": -10.096076011657715, + "step": 2870 + }, + { + "epoch": 2.99, + "learning_rate": 1.7307692307692307e-09, + "logits/chosen": -2.3136038780212402, + "logits/rejected": -2.09283447265625, + "logps/chosen": -289.9815368652344, + "logps/rejected": -315.29498291015625, + "loss": 0.0066, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.966369867324829, + "rewards/margins": 7.501823425292969, + "rewards/rejected": -9.468193054199219, + "step": 2880 + }, + { + "epoch": 3.0, + "step": 2889, + "total_flos": 0.0, + "train_loss": 0.16336456995732068, + "train_runtime": 43274.0653, + "train_samples_per_second": 4.272, + "train_steps_per_second": 0.067 + } + ], + "logging_steps": 10, + "max_steps": 2889, + "num_train_epochs": 3, + "save_steps": 500, + "total_flos": 0.0, + "trial_name": null, + "trial_params": null +}