| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9984, | |
| "eval_steps": 100, | |
| "global_step": 312, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.5625e-07, | |
| "logits/chosen": -0.8750922679901123, | |
| "logits/rejected": -0.7979680895805359, | |
| "logps/chosen": -60.00876235961914, | |
| "logps/rejected": -56.38082504272461, | |
| "loss": 0.1482, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 1.5625e-06, | |
| "logits/chosen": -1.0051445960998535, | |
| "logits/rejected": -0.9556077122688293, | |
| "logps/chosen": -30.64621925354004, | |
| "logps/rejected": -24.19979476928711, | |
| "loss": 0.1565, | |
| "rewards/accuracies": 0.02083333395421505, | |
| "rewards/chosen": -8.15845123725012e-05, | |
| "rewards/margins": -4.479259951040149e-05, | |
| "rewards/rejected": -3.679191286209971e-05, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 3.125e-06, | |
| "logits/chosen": -1.0498403310775757, | |
| "logits/rejected": -0.992475152015686, | |
| "logps/chosen": -40.17894744873047, | |
| "logps/rejected": -31.744924545288086, | |
| "loss": 0.1421, | |
| "rewards/accuracies": 0.03125, | |
| "rewards/chosen": -3.2624266168568283e-06, | |
| "rewards/margins": -6.817355460952967e-05, | |
| "rewards/rejected": 6.491113163065165e-05, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 4.6875000000000004e-06, | |
| "logits/chosen": -1.0504692792892456, | |
| "logits/rejected": -1.0281661748886108, | |
| "logps/chosen": -23.273496627807617, | |
| "logps/rejected": -18.378360748291016, | |
| "loss": 0.1594, | |
| "rewards/accuracies": 0.01875000074505806, | |
| "rewards/chosen": -7.991820166353136e-05, | |
| "rewards/margins": -0.00010900965571636334, | |
| "rewards/rejected": 2.909145587182138e-05, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 4.989935734988098e-06, | |
| "logits/chosen": -1.0432171821594238, | |
| "logits/rejected": -1.0193238258361816, | |
| "logps/chosen": -14.588598251342773, | |
| "logps/rejected": -13.306127548217773, | |
| "loss": 0.1303, | |
| "rewards/accuracies": 0.012500000186264515, | |
| "rewards/chosen": 4.2490195482969284e-05, | |
| "rewards/margins": -2.1462084987433627e-05, | |
| "rewards/rejected": 6.39522768324241e-05, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 4.949188496058089e-06, | |
| "logits/chosen": -1.0302413702011108, | |
| "logits/rejected": -0.986968994140625, | |
| "logps/chosen": -9.379155158996582, | |
| "logps/rejected": -9.404864311218262, | |
| "loss": 0.1469, | |
| "rewards/accuracies": 0.02500000037252903, | |
| "rewards/chosen": 3.0187036827555858e-05, | |
| "rewards/margins": 4.167082079220563e-05, | |
| "rewards/rejected": -1.1483787602628581e-05, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 4.8776412907378845e-06, | |
| "logits/chosen": -0.9920421838760376, | |
| "logits/rejected": -0.9573459625244141, | |
| "logps/chosen": -17.296560287475586, | |
| "logps/rejected": -15.45374584197998, | |
| "loss": 0.1519, | |
| "rewards/accuracies": 0.012500000186264515, | |
| "rewards/chosen": 2.950535053969361e-05, | |
| "rewards/margins": -1.730041731207166e-05, | |
| "rewards/rejected": 4.680576239479706e-05, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 4.7761938666470405e-06, | |
| "logits/chosen": -1.0576204061508179, | |
| "logits/rejected": -1.0229989290237427, | |
| "logps/chosen": -30.714122772216797, | |
| "logps/rejected": -24.89596939086914, | |
| "loss": 0.1346, | |
| "rewards/accuracies": 0.05000000074505806, | |
| "rewards/chosen": 0.00010937159095192328, | |
| "rewards/margins": -1.5012978110462427e-05, | |
| "rewards/rejected": 0.0001243845617864281, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 4.646121984004666e-06, | |
| "logits/chosen": -1.0434526205062866, | |
| "logits/rejected": -1.0283691883087158, | |
| "logps/chosen": -12.912440299987793, | |
| "logps/rejected": -10.21430778503418, | |
| "loss": 0.1494, | |
| "rewards/accuracies": 0.01875000074505806, | |
| "rewards/chosen": 6.350317562464625e-05, | |
| "rewards/margins": 3.430190190556459e-05, | |
| "rewards/rejected": 2.9201270081102848e-05, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 4.4890613722044526e-06, | |
| "logits/chosen": -1.005961298942566, | |
| "logits/rejected": -0.9507063627243042, | |
| "logps/chosen": -29.203853607177734, | |
| "logps/rejected": -24.27176856994629, | |
| "loss": 0.1563, | |
| "rewards/accuracies": 0.02500000037252903, | |
| "rewards/chosen": 0.00019098483608104289, | |
| "rewards/margins": -0.00010294451203662902, | |
| "rewards/rejected": 0.00029392936266958714, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 4.3069871595684795e-06, | |
| "logits/chosen": -1.0817575454711914, | |
| "logits/rejected": -1.0109978914260864, | |
| "logps/chosen": -23.601749420166016, | |
| "logps/rejected": -21.154720306396484, | |
| "loss": 0.1204, | |
| "rewards/accuracies": 0.03750000149011612, | |
| "rewards/chosen": 0.0001659760600887239, | |
| "rewards/margins": 1.1245434507145546e-05, | |
| "rewards/rejected": 0.00015473061648663133, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "eval_logits/chosen": -0.8344239592552185, | |
| "eval_logits/rejected": -0.7404021620750427, | |
| "eval_logps/chosen": -395.97412109375, | |
| "eval_logps/rejected": -366.70782470703125, | |
| "eval_loss": 0.05379165709018707, | |
| "eval_rewards/accuracies": 0.4399999976158142, | |
| "eval_rewards/chosen": 0.0003311456530354917, | |
| "eval_rewards/margins": -0.0001964901020983234, | |
| "eval_rewards/rejected": 0.0005276357405818999, | |
| "eval_runtime": 545.7797, | |
| "eval_samples_per_second": 3.664, | |
| "eval_steps_per_second": 0.916, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 4.102189034962561e-06, | |
| "logits/chosen": -1.0390229225158691, | |
| "logits/rejected": -1.025102972984314, | |
| "logps/chosen": -21.16196060180664, | |
| "logps/rejected": -22.110755920410156, | |
| "loss": 0.1488, | |
| "rewards/accuracies": 0.03125, | |
| "rewards/chosen": 0.00030253888689912856, | |
| "rewards/margins": 5.533280273084529e-05, | |
| "rewards/rejected": 0.0002472061023581773, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 3.8772424536302565e-06, | |
| "logits/chosen": -1.029483437538147, | |
| "logits/rejected": -0.9574755430221558, | |
| "logps/chosen": -19.494661331176758, | |
| "logps/rejected": -17.480262756347656, | |
| "loss": 0.145, | |
| "rewards/accuracies": 0.04374999925494194, | |
| "rewards/chosen": 0.0003196864854544401, | |
| "rewards/margins": 0.00018756816280074418, | |
| "rewards/rejected": 0.00013211832265369594, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 3.634976249348867e-06, | |
| "logits/chosen": -1.107649564743042, | |
| "logits/rejected": -1.076535701751709, | |
| "logps/chosen": -11.74854850769043, | |
| "logps/rejected": -9.875129699707031, | |
| "loss": 0.1365, | |
| "rewards/accuracies": 0.01875000074505806, | |
| "rewards/chosen": 0.00019255562801845372, | |
| "rewards/margins": 3.773098433157429e-05, | |
| "rewards/rejected": 0.0001548246364109218, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 3.3784370602033572e-06, | |
| "logits/chosen": -1.0260846614837646, | |
| "logits/rejected": -0.9438373446464539, | |
| "logps/chosen": -17.046716690063477, | |
| "logps/rejected": -12.897377014160156, | |
| "loss": 0.1655, | |
| "rewards/accuracies": 0.03125, | |
| "rewards/chosen": 0.00021877660765312612, | |
| "rewards/margins": 0.00010056849714601412, | |
| "rewards/rejected": 0.00011820811778306961, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 3.1108510153447352e-06, | |
| "logits/chosen": -1.0065504312515259, | |
| "logits/rejected": -0.9879522323608398, | |
| "logps/chosen": -14.441717147827148, | |
| "logps/rejected": -11.689096450805664, | |
| "loss": 0.1525, | |
| "rewards/accuracies": 0.02500000037252903, | |
| "rewards/chosen": 0.00024849834153428674, | |
| "rewards/margins": 9.472499368712306e-05, | |
| "rewards/rejected": 0.0001537733623990789, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 2.835583164544139e-06, | |
| "logits/chosen": -1.0789475440979004, | |
| "logits/rejected": -1.0655437707901, | |
| "logps/chosen": -14.895776748657227, | |
| "logps/rejected": -14.265124320983887, | |
| "loss": 0.1408, | |
| "rewards/accuracies": 0.02500000037252903, | |
| "rewards/chosen": 0.00023004882677923888, | |
| "rewards/margins": -1.2942269677296281e-05, | |
| "rewards/rejected": 0.0002429911110084504, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 2.556095160739513e-06, | |
| "logits/chosen": -1.059157371520996, | |
| "logits/rejected": -0.9842355847358704, | |
| "logps/chosen": -23.611387252807617, | |
| "logps/rejected": -22.636394500732422, | |
| "loss": 0.1427, | |
| "rewards/accuracies": 0.03125, | |
| "rewards/chosen": 0.00042936400859616697, | |
| "rewards/margins": 1.9614741177065298e-05, | |
| "rewards/rejected": 0.0004097492783330381, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 2.2759017277414165e-06, | |
| "logits/chosen": -1.0258867740631104, | |
| "logits/rejected": -0.9551021456718445, | |
| "logps/chosen": -26.574390411376953, | |
| "logps/rejected": -17.66288948059082, | |
| "loss": 0.1366, | |
| "rewards/accuracies": 0.02500000037252903, | |
| "rewards/chosen": 0.0002309099363628775, | |
| "rewards/margins": -0.00013474392471835017, | |
| "rewards/rejected": 0.00036565386108122766, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 1.9985264605418185e-06, | |
| "logits/chosen": -1.0549209117889404, | |
| "logits/rejected": -1.0101747512817383, | |
| "logps/chosen": -13.79809284210205, | |
| "logps/rejected": -12.155397415161133, | |
| "loss": 0.1457, | |
| "rewards/accuracies": 0.012500000186264515, | |
| "rewards/chosen": 0.00018917236593551934, | |
| "rewards/margins": -1.1611557965807151e-05, | |
| "rewards/rejected": 0.00020078392117284238, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 1.7274575140626318e-06, | |
| "logits/chosen": -1.0925042629241943, | |
| "logits/rejected": -1.0523203611373901, | |
| "logps/chosen": -22.70647621154785, | |
| "logps/rejected": -21.175107955932617, | |
| "loss": 0.1673, | |
| "rewards/accuracies": 0.02500000037252903, | |
| "rewards/chosen": 0.00025192610337398946, | |
| "rewards/margins": 6.463247700594366e-06, | |
| "rewards/rejected": 0.0002454628120176494, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "eval_logits/chosen": -0.8413296937942505, | |
| "eval_logits/rejected": -0.7451605796813965, | |
| "eval_logps/chosen": -394.95245361328125, | |
| "eval_logps/rejected": -365.8174743652344, | |
| "eval_loss": 0.05375674366950989, | |
| "eval_rewards/accuracies": 0.4465000033378601, | |
| "eval_rewards/chosen": 0.0013527950504794717, | |
| "eval_rewards/margins": -6.51527734589763e-05, | |
| "eval_rewards/rejected": 0.0014179478166624904, | |
| "eval_runtime": 545.9031, | |
| "eval_samples_per_second": 3.664, | |
| "eval_steps_per_second": 0.916, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 1.466103737583699e-06, | |
| "logits/chosen": -1.0759646892547607, | |
| "logits/rejected": -1.0574228763580322, | |
| "logps/chosen": -21.28565216064453, | |
| "logps/rejected": -18.393810272216797, | |
| "loss": 0.1471, | |
| "rewards/accuracies": 0.01875000074505806, | |
| "rewards/chosen": 0.0002163530298275873, | |
| "rewards/margins": -0.00010527259291848168, | |
| "rewards/rejected": 0.0003216256445739418, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 1.217751806485235e-06, | |
| "logits/chosen": -1.036292314529419, | |
| "logits/rejected": -0.9575215578079224, | |
| "logps/chosen": -19.784107208251953, | |
| "logps/rejected": -16.992115020751953, | |
| "loss": 0.1377, | |
| "rewards/accuracies": 0.03125, | |
| "rewards/chosen": 0.00023899655207060277, | |
| "rewards/margins": 5.950551712885499e-05, | |
| "rewards/rejected": 0.000179491049493663, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 9.855248903979505e-07, | |
| "logits/chosen": -1.042037844657898, | |
| "logits/rejected": -1.0194261074066162, | |
| "logps/chosen": -29.382129669189453, | |
| "logps/rejected": -24.937179565429688, | |
| "loss": 0.1457, | |
| "rewards/accuracies": 0.05000000074505806, | |
| "rewards/chosen": 0.0005346836405806243, | |
| "rewards/margins": 4.8290185077348724e-05, | |
| "rewards/rejected": 0.0004863934591412544, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 7.723433775328385e-07, | |
| "logits/chosen": -1.0891209840774536, | |
| "logits/rejected": -1.0485570430755615, | |
| "logps/chosen": -24.730152130126953, | |
| "logps/rejected": -23.654470443725586, | |
| "loss": 0.157, | |
| "rewards/accuracies": 0.03125, | |
| "rewards/chosen": 0.00022753537632524967, | |
| "rewards/margins": -0.00011534886289155111, | |
| "rewards/rejected": 0.0003428841882850975, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 5.808881491049723e-07, | |
| "logits/chosen": -1.1149171590805054, | |
| "logits/rejected": -1.0429071187973022, | |
| "logps/chosen": -38.763633728027344, | |
| "logps/rejected": -27.425064086914062, | |
| "loss": 0.1685, | |
| "rewards/accuracies": 0.05000000074505806, | |
| "rewards/chosen": 0.0005215436685830355, | |
| "rewards/margins": 2.6613433874445036e-05, | |
| "rewards/rejected": 0.0004949303111061454, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 4.1356686569674344e-07, | |
| "logits/chosen": -1.0859912633895874, | |
| "logits/rejected": -1.0464859008789062, | |
| "logps/chosen": -27.60003662109375, | |
| "logps/rejected": -23.53853988647461, | |
| "loss": 0.1439, | |
| "rewards/accuracies": 0.03125, | |
| "rewards/chosen": 0.000496146094519645, | |
| "rewards/margins": 2.9071514290990308e-05, | |
| "rewards/rejected": 0.00046707456931471825, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 2.7248368952908055e-07, | |
| "logits/chosen": -1.0678870677947998, | |
| "logits/rejected": -1.0435296297073364, | |
| "logps/chosen": -32.16819763183594, | |
| "logps/rejected": -28.180551528930664, | |
| "loss": 0.1724, | |
| "rewards/accuracies": 0.03750000149011612, | |
| "rewards/chosen": 0.0004276956315152347, | |
| "rewards/margins": 2.2596535927732475e-05, | |
| "rewards/rejected": 0.00040509915561415255, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 1.59412823400657e-07, | |
| "logits/chosen": -1.0548503398895264, | |
| "logits/rejected": -1.0008891820907593, | |
| "logps/chosen": -25.87237548828125, | |
| "logps/rejected": -24.495941162109375, | |
| "loss": 0.1347, | |
| "rewards/accuracies": 0.03750000149011612, | |
| "rewards/chosen": 0.0003715341736096889, | |
| "rewards/margins": -5.351124127628282e-05, | |
| "rewards/rejected": 0.00042504543671384454, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 7.577619905828281e-08, | |
| "logits/chosen": -1.0749586820602417, | |
| "logits/rejected": -1.0013477802276611, | |
| "logps/chosen": -24.45828628540039, | |
| "logps/rejected": -17.479082107543945, | |
| "loss": 0.1346, | |
| "rewards/accuracies": 0.03125, | |
| "rewards/chosen": 0.0003076163411606103, | |
| "rewards/margins": 9.37794684432447e-05, | |
| "rewards/rejected": 0.00021383687271736562, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.262559558016325e-08, | |
| "logits/chosen": -1.0697089433670044, | |
| "logits/rejected": -1.0091029405593872, | |
| "logps/chosen": -17.250530242919922, | |
| "logps/rejected": -16.223955154418945, | |
| "loss": 0.1395, | |
| "rewards/accuracies": 0.01875000074505806, | |
| "rewards/chosen": 0.00032176534296013415, | |
| "rewards/margins": -2.4562166800023988e-05, | |
| "rewards/rejected": 0.00034632752067409456, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "eval_logits/chosen": -0.8510691523551941, | |
| "eval_logits/rejected": -0.7548762559890747, | |
| "eval_logps/chosen": -394.70794677734375, | |
| "eval_logps/rejected": -365.4079284667969, | |
| "eval_loss": 0.053816698491573334, | |
| "eval_rewards/accuracies": 0.4410000145435333, | |
| "eval_rewards/chosen": 0.0015973460394889116, | |
| "eval_rewards/margins": -0.0002301189670106396, | |
| "eval_rewards/rejected": 0.0018274650210514665, | |
| "eval_runtime": 545.8177, | |
| "eval_samples_per_second": 3.664, | |
| "eval_steps_per_second": 0.916, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 6.294126437336734e-10, | |
| "logits/chosen": -1.0546523332595825, | |
| "logits/rejected": -0.9914261698722839, | |
| "logps/chosen": -20.59957504272461, | |
| "logps/rejected": -15.304788589477539, | |
| "loss": 0.1463, | |
| "rewards/accuracies": 0.02500000037252903, | |
| "rewards/chosen": 0.0003310924512334168, | |
| "rewards/margins": 6.335419311653823e-05, | |
| "rewards/rejected": 0.00026773830177262425, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 312, | |
| "total_flos": 0.0, | |
| "train_loss": 0.14698060888510484, | |
| "train_runtime": 3810.9697, | |
| "train_samples_per_second": 1.312, | |
| "train_steps_per_second": 0.082 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 312, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |