|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 1.0, |
|
"eval_steps": 100, |
|
"global_step": 327, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0, |
|
"grad_norm": 37.56032423344454, |
|
"learning_rate": 1.5151515151515152e-08, |
|
"logits/chosen": -3.115293502807617, |
|
"logits/rejected": -3.0863265991210938, |
|
"logps/chosen": -870.7420043945312, |
|
"logps/rejected": -1334.502685546875, |
|
"loss": 0.6931, |
|
"rewards/accuracies": 0.0, |
|
"rewards/chosen": 0.0, |
|
"rewards/margins": 0.0, |
|
"rewards/rejected": 0.0, |
|
"step": 1 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"grad_norm": 37.49331944203386, |
|
"learning_rate": 1.5151515151515152e-07, |
|
"logits/chosen": -3.0526609420776367, |
|
"logits/rejected": -3.0749807357788086, |
|
"logps/chosen": -940.7243041992188, |
|
"logps/rejected": -1276.6510009765625, |
|
"loss": 0.6922, |
|
"rewards/accuracies": 0.5486111044883728, |
|
"rewards/chosen": 0.001147550530731678, |
|
"rewards/margins": 0.0018619541078805923, |
|
"rewards/rejected": -0.000714403809979558, |
|
"step": 10 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"grad_norm": 32.798777633158615, |
|
"learning_rate": 3.0303030303030305e-07, |
|
"logits/chosen": -3.0576233863830566, |
|
"logits/rejected": -3.0690040588378906, |
|
"logps/chosen": -956.8638916015625, |
|
"logps/rejected": -1231.0487060546875, |
|
"loss": 0.668, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": 0.03549911081790924, |
|
"rewards/margins": 0.057550616562366486, |
|
"rewards/rejected": -0.022051498293876648, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"grad_norm": 31.200374512571734, |
|
"learning_rate": 4.545454545454545e-07, |
|
"logits/chosen": -3.1981358528137207, |
|
"logits/rejected": -3.1817786693573, |
|
"logps/chosen": -1004.2825927734375, |
|
"logps/rejected": -1336.2808837890625, |
|
"loss": 0.6056, |
|
"rewards/accuracies": 0.800000011920929, |
|
"rewards/chosen": -0.0012753400951623917, |
|
"rewards/margins": 0.2531731426715851, |
|
"rewards/rejected": -0.25444847345352173, |
|
"step": 30 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"grad_norm": 31.57664285278684, |
|
"learning_rate": 4.993009492952949e-07, |
|
"logits/chosen": -3.2853000164031982, |
|
"logits/rejected": -3.300828218460083, |
|
"logps/chosen": -1001.1776123046875, |
|
"logps/rejected": -1323.44287109375, |
|
"loss": 0.5156, |
|
"rewards/accuracies": 0.8125, |
|
"rewards/chosen": -0.10599758476018906, |
|
"rewards/margins": 0.550561785697937, |
|
"rewards/rejected": -0.656559407711029, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"grad_norm": 32.0572257728948, |
|
"learning_rate": 4.958864353026382e-07, |
|
"logits/chosen": -3.3941280841827393, |
|
"logits/rejected": -3.3549060821533203, |
|
"logps/chosen": -880.5036010742188, |
|
"logps/rejected": -1494.1220703125, |
|
"loss": 0.4348, |
|
"rewards/accuracies": 0.8812500238418579, |
|
"rewards/chosen": -0.06764356046915054, |
|
"rewards/margins": 1.6453561782836914, |
|
"rewards/rejected": -1.712999701499939, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"grad_norm": 37.653606745673876, |
|
"learning_rate": 4.896669632591651e-07, |
|
"logits/chosen": -3.4180691242218018, |
|
"logits/rejected": -3.3949356079101562, |
|
"logps/chosen": -997.1550903320312, |
|
"logps/rejected": -1486.9512939453125, |
|
"loss": 0.4241, |
|
"rewards/accuracies": 0.84375, |
|
"rewards/chosen": -0.191396564245224, |
|
"rewards/margins": 1.6561005115509033, |
|
"rewards/rejected": -1.847496747970581, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"grad_norm": 27.62213668513971, |
|
"learning_rate": 4.80713482008226e-07, |
|
"logits/chosen": -3.2756118774414062, |
|
"logits/rejected": -3.325742721557617, |
|
"logps/chosen": -1002.0143432617188, |
|
"logps/rejected": -1562.6756591796875, |
|
"loss": 0.3597, |
|
"rewards/accuracies": 0.8062499761581421, |
|
"rewards/chosen": -0.11527639627456665, |
|
"rewards/margins": 2.851576566696167, |
|
"rewards/rejected": -2.966853380203247, |
|
"step": 70 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"grad_norm": 24.96995353325617, |
|
"learning_rate": 4.691281286981619e-07, |
|
"logits/chosen": -3.2543506622314453, |
|
"logits/rejected": -3.2743306159973145, |
|
"logps/chosen": -1047.9954833984375, |
|
"logps/rejected": -1567.3741455078125, |
|
"loss": 0.3576, |
|
"rewards/accuracies": 0.84375, |
|
"rewards/chosen": -0.11167576164007187, |
|
"rewards/margins": 2.7660152912139893, |
|
"rewards/rejected": -2.8776910305023193, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"grad_norm": 27.750313403051702, |
|
"learning_rate": 4.550430636492389e-07, |
|
"logits/chosen": -3.152367115020752, |
|
"logits/rejected": -3.1291539669036865, |
|
"logps/chosen": -958.4727783203125, |
|
"logps/rejected": -1616.8697509765625, |
|
"loss": 0.3037, |
|
"rewards/accuracies": 0.887499988079071, |
|
"rewards/chosen": -0.08375158160924911, |
|
"rewards/margins": 3.5008788108825684, |
|
"rewards/rejected": -3.5846304893493652, |
|
"step": 90 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"grad_norm": 30.508104431701753, |
|
"learning_rate": 4.386189627302036e-07, |
|
"logits/chosen": -3.1359643936157227, |
|
"logits/rejected": -3.0350069999694824, |
|
"logps/chosen": -1068.0445556640625, |
|
"logps/rejected": -1752.9798583984375, |
|
"loss": 0.2478, |
|
"rewards/accuracies": 0.8999999761581421, |
|
"rewards/chosen": -0.297347754240036, |
|
"rewards/margins": 4.142968654632568, |
|
"rewards/rejected": -4.440316200256348, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"eval_logits/chosen": -2.116163492202759, |
|
"eval_logits/rejected": -2.9199953079223633, |
|
"eval_logps/chosen": -214.01956176757812, |
|
"eval_logps/rejected": -996.0294189453125, |
|
"eval_loss": 0.03523317724466324, |
|
"eval_rewards/accuracies": 1.0, |
|
"eval_rewards/chosen": -0.1739114671945572, |
|
"eval_rewards/margins": 4.2524919509887695, |
|
"eval_rewards/rejected": -4.426403999328613, |
|
"eval_runtime": 6.9945, |
|
"eval_samples_per_second": 9.15, |
|
"eval_steps_per_second": 0.286, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"grad_norm": 30.142384194174287, |
|
"learning_rate": 4.200431844427298e-07, |
|
"logits/chosen": -3.1088919639587402, |
|
"logits/rejected": -2.9363715648651123, |
|
"logps/chosen": -956.9268798828125, |
|
"logps/rejected": -1825.817138671875, |
|
"loss": 0.2479, |
|
"rewards/accuracies": 0.925000011920929, |
|
"rewards/chosen": -0.4095672070980072, |
|
"rewards/margins": 4.6759538650512695, |
|
"rewards/rejected": -5.085520267486572, |
|
"step": 110 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"grad_norm": 30.548015460707564, |
|
"learning_rate": 3.9952763262280397e-07, |
|
"logits/chosen": -3.0125510692596436, |
|
"logits/rejected": -2.8323769569396973, |
|
"logps/chosen": -974.1087646484375, |
|
"logps/rejected": -1928.265380859375, |
|
"loss": 0.2222, |
|
"rewards/accuracies": 0.90625, |
|
"rewards/chosen": -0.6024614572525024, |
|
"rewards/margins": 5.620959281921387, |
|
"rewards/rejected": -6.223421096801758, |
|
"step": 120 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"grad_norm": 23.95507006913498, |
|
"learning_rate": 3.7730633914035584e-07, |
|
"logits/chosen": -2.9912257194519043, |
|
"logits/rejected": -2.7818751335144043, |
|
"logps/chosen": -1056.4798583984375, |
|
"logps/rejected": -2160.14990234375, |
|
"loss": 0.1988, |
|
"rewards/accuracies": 0.949999988079071, |
|
"rewards/chosen": -0.6903690099716187, |
|
"rewards/margins": 7.284422397613525, |
|
"rewards/rejected": -7.974791526794434, |
|
"step": 130 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"grad_norm": 42.81780370407222, |
|
"learning_rate": 3.5363279417257385e-07, |
|
"logits/chosen": -2.8955721855163574, |
|
"logits/rejected": -2.754878044128418, |
|
"logps/chosen": -1038.2923583984375, |
|
"logps/rejected": -1966.652587890625, |
|
"loss": 0.2092, |
|
"rewards/accuracies": 0.90625, |
|
"rewards/chosen": -1.0339103937149048, |
|
"rewards/margins": 5.819826126098633, |
|
"rewards/rejected": -6.85373592376709, |
|
"step": 140 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"grad_norm": 31.239186243610785, |
|
"learning_rate": 3.287770545059052e-07, |
|
"logits/chosen": -2.8968818187713623, |
|
"logits/rejected": -2.668034076690674, |
|
"logps/chosen": -1137.552978515625, |
|
"logps/rejected": -2158.06298828125, |
|
"loss": 0.1807, |
|
"rewards/accuracies": 0.949999988079071, |
|
"rewards/chosen": -1.3718111515045166, |
|
"rewards/margins": 7.208723545074463, |
|
"rewards/rejected": -8.580533981323242, |
|
"step": 150 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"grad_norm": 30.783011951421607, |
|
"learning_rate": 3.030226628538842e-07, |
|
"logits/chosen": -2.7924959659576416, |
|
"logits/rejected": -2.620363473892212, |
|
"logps/chosen": -1211.665771484375, |
|
"logps/rejected": -2208.79833984375, |
|
"loss": 0.1596, |
|
"rewards/accuracies": 0.949999988079071, |
|
"rewards/chosen": -1.6732728481292725, |
|
"rewards/margins": 7.373621463775635, |
|
"rewards/rejected": -9.046895027160645, |
|
"step": 160 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"grad_norm": 38.33986073767423, |
|
"learning_rate": 2.7666341333378004e-07, |
|
"logits/chosen": -2.7492575645446777, |
|
"logits/rejected": -2.5758957862854004, |
|
"logps/chosen": -1068.8045654296875, |
|
"logps/rejected": -2013.82421875, |
|
"loss": 0.1765, |
|
"rewards/accuracies": 0.9624999761581421, |
|
"rewards/chosen": -1.2596815824508667, |
|
"rewards/margins": 6.303467750549316, |
|
"rewards/rejected": -7.563148498535156, |
|
"step": 170 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"grad_norm": 46.105813888725216, |
|
"learning_rate": 2.5e-07, |
|
"logits/chosen": -2.7488133907318115, |
|
"logits/rejected": -2.492980480194092, |
|
"logps/chosen": -1113.6063232421875, |
|
"logps/rejected": -2361.96923828125, |
|
"loss": 0.1225, |
|
"rewards/accuracies": 0.956250011920929, |
|
"rewards/chosen": -1.5648447275161743, |
|
"rewards/margins": 8.788183212280273, |
|
"rewards/rejected": -10.353028297424316, |
|
"step": 180 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"grad_norm": 38.659892201318485, |
|
"learning_rate": 2.2333658666621994e-07, |
|
"logits/chosen": -2.716352939605713, |
|
"logits/rejected": -2.4929823875427246, |
|
"logps/chosen": -1151.252685546875, |
|
"logps/rejected": -2195.82958984375, |
|
"loss": 0.1587, |
|
"rewards/accuracies": 0.956250011920929, |
|
"rewards/chosen": -1.521728277206421, |
|
"rewards/margins": 7.973954677581787, |
|
"rewards/rejected": -9.495683670043945, |
|
"step": 190 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"grad_norm": 34.038321670255556, |
|
"learning_rate": 1.9697733714611587e-07, |
|
"logits/chosen": -2.77030873298645, |
|
"logits/rejected": -2.5207982063293457, |
|
"logps/chosen": -1151.6956787109375, |
|
"logps/rejected": -2134.08349609375, |
|
"loss": 0.1385, |
|
"rewards/accuracies": 0.956250011920929, |
|
"rewards/chosen": -1.476732850074768, |
|
"rewards/margins": 6.9633917808532715, |
|
"rewards/rejected": -8.440125465393066, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"eval_logits/chosen": -1.6287028789520264, |
|
"eval_logits/rejected": -2.333620548248291, |
|
"eval_logps/chosen": -230.22816467285156, |
|
"eval_logps/rejected": -1373.3590087890625, |
|
"eval_loss": 0.00411956338211894, |
|
"eval_rewards/accuracies": 1.0, |
|
"eval_rewards/chosen": -0.3359977602958679, |
|
"eval_rewards/margins": 7.863701820373535, |
|
"eval_rewards/rejected": -8.199699401855469, |
|
"eval_runtime": 6.9561, |
|
"eval_samples_per_second": 9.201, |
|
"eval_steps_per_second": 0.288, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"grad_norm": 25.859030241191462, |
|
"learning_rate": 1.7122294549409482e-07, |
|
"logits/chosen": -2.7445123195648193, |
|
"logits/rejected": -2.477301597595215, |
|
"logps/chosen": -1209.62158203125, |
|
"logps/rejected": -2304.346435546875, |
|
"loss": 0.1239, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": -1.5678670406341553, |
|
"rewards/margins": 8.935761451721191, |
|
"rewards/rejected": -10.503628730773926, |
|
"step": 210 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"grad_norm": 54.09588414480716, |
|
"learning_rate": 1.4636720582742613e-07, |
|
"logits/chosen": -2.6386165618896484, |
|
"logits/rejected": -2.3536062240600586, |
|
"logps/chosen": -1170.019287109375, |
|
"logps/rejected": -2326.010986328125, |
|
"loss": 0.1338, |
|
"rewards/accuracies": 0.949999988079071, |
|
"rewards/chosen": -1.9397300481796265, |
|
"rewards/margins": 8.54932975769043, |
|
"rewards/rejected": -10.489059448242188, |
|
"step": 220 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"grad_norm": 35.229208104251256, |
|
"learning_rate": 1.2269366085964422e-07, |
|
"logits/chosen": -2.594449758529663, |
|
"logits/rejected": -2.3216192722320557, |
|
"logps/chosen": -1134.5150146484375, |
|
"logps/rejected": -2302.87158203125, |
|
"loss": 0.1241, |
|
"rewards/accuracies": 0.956250011920929, |
|
"rewards/chosen": -2.0043346881866455, |
|
"rewards/margins": 8.11410903930664, |
|
"rewards/rejected": -10.118444442749023, |
|
"step": 230 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"grad_norm": 35.55573135898996, |
|
"learning_rate": 1.00472367377196e-07, |
|
"logits/chosen": -2.6050658226013184, |
|
"logits/rejected": -2.3415164947509766, |
|
"logps/chosen": -1212.265380859375, |
|
"logps/rejected": -2304.11669921875, |
|
"loss": 0.1046, |
|
"rewards/accuracies": 0.956250011920929, |
|
"rewards/chosen": -1.8809270858764648, |
|
"rewards/margins": 8.250687599182129, |
|
"rewards/rejected": -10.131612777709961, |
|
"step": 240 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"grad_norm": 31.88590245463047, |
|
"learning_rate": 7.99568155572701e-08, |
|
"logits/chosen": -2.6101841926574707, |
|
"logits/rejected": -2.0731587409973145, |
|
"logps/chosen": -1139.8570556640625, |
|
"logps/rejected": -2845.157470703125, |
|
"loss": 0.0885, |
|
"rewards/accuracies": 0.9624999761581421, |
|
"rewards/chosen": -1.8957105875015259, |
|
"rewards/margins": 13.620719909667969, |
|
"rewards/rejected": -15.516430854797363, |
|
"step": 250 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"grad_norm": 51.7448993159194, |
|
"learning_rate": 6.13810372697964e-08, |
|
"logits/chosen": -2.5568299293518066, |
|
"logits/rejected": -2.152559757232666, |
|
"logps/chosen": -1236.4830322265625, |
|
"logps/rejected": -2579.609619140625, |
|
"loss": 0.1114, |
|
"rewards/accuracies": 0.956250011920929, |
|
"rewards/chosen": -2.0548171997070312, |
|
"rewards/margins": 11.762044906616211, |
|
"rewards/rejected": -13.816862106323242, |
|
"step": 260 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"grad_norm": 27.4784440806591, |
|
"learning_rate": 4.4956936350761005e-08, |
|
"logits/chosen": -2.598968267440796, |
|
"logits/rejected": -2.217064619064331, |
|
"logps/chosen": -1198.1903076171875, |
|
"logps/rejected": -2465.025634765625, |
|
"loss": 0.0966, |
|
"rewards/accuracies": 0.9750000238418579, |
|
"rewards/chosen": -2.0705723762512207, |
|
"rewards/margins": 9.766708374023438, |
|
"rewards/rejected": -11.837281227111816, |
|
"step": 270 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"grad_norm": 35.99094553945511, |
|
"learning_rate": 3.087187130183805e-08, |
|
"logits/chosen": -2.51914119720459, |
|
"logits/rejected": -2.214329481124878, |
|
"logps/chosen": -1253.900146484375, |
|
"logps/rejected": -2469.315185546875, |
|
"loss": 0.1108, |
|
"rewards/accuracies": 0.9750000238418579, |
|
"rewards/chosen": -2.140868663787842, |
|
"rewards/margins": 9.513227462768555, |
|
"rewards/rejected": -11.654096603393555, |
|
"step": 280 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"grad_norm": 62.565118319363144, |
|
"learning_rate": 1.928651799177408e-08, |
|
"logits/chosen": -2.5865471363067627, |
|
"logits/rejected": -2.248619556427002, |
|
"logps/chosen": -1134.03173828125, |
|
"logps/rejected": -2389.395751953125, |
|
"loss": 0.1085, |
|
"rewards/accuracies": 0.96875, |
|
"rewards/chosen": -2.002607822418213, |
|
"rewards/margins": 8.705282211303711, |
|
"rewards/rejected": -10.707889556884766, |
|
"step": 290 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"grad_norm": 37.73877960542491, |
|
"learning_rate": 1.0333036740834855e-08, |
|
"logits/chosen": -2.552736520767212, |
|
"logits/rejected": -2.2399215698242188, |
|
"logps/chosen": -1204.527587890625, |
|
"logps/rejected": -2376.899169921875, |
|
"loss": 0.0899, |
|
"rewards/accuracies": 0.9937499761581421, |
|
"rewards/chosen": -2.1047260761260986, |
|
"rewards/margins": 8.565834045410156, |
|
"rewards/rejected": -10.670559883117676, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"eval_logits/chosen": -1.3416023254394531, |
|
"eval_logits/rejected": -2.055326461791992, |
|
"eval_logps/chosen": -241.41647338867188, |
|
"eval_logps/rejected": -1559.6263427734375, |
|
"eval_loss": 0.0018886247416958213, |
|
"eval_rewards/accuracies": 1.0, |
|
"eval_rewards/chosen": -0.44788068532943726, |
|
"eval_rewards/margins": 9.614492416381836, |
|
"eval_rewards/rejected": -10.062373161315918, |
|
"eval_runtime": 6.9591, |
|
"eval_samples_per_second": 9.197, |
|
"eval_steps_per_second": 0.287, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"grad_norm": 29.76747785442669, |
|
"learning_rate": 4.113564697361821e-09, |
|
"logits/chosen": -2.521284580230713, |
|
"logits/rejected": -2.1441428661346436, |
|
"logps/chosen": -1216.38916015625, |
|
"logps/rejected": -2597.3994140625, |
|
"loss": 0.0804, |
|
"rewards/accuracies": 0.9624999761581421, |
|
"rewards/chosen": -1.9038699865341187, |
|
"rewards/margins": 11.402238845825195, |
|
"rewards/rejected": -13.306109428405762, |
|
"step": 310 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"grad_norm": 32.64158882779933, |
|
"learning_rate": 6.990507047049676e-10, |
|
"logits/chosen": -2.489298105239868, |
|
"logits/rejected": -2.159824848175049, |
|
"logps/chosen": -1119.7908935546875, |
|
"logps/rejected": -2548.427001953125, |
|
"loss": 0.0939, |
|
"rewards/accuracies": 0.9624999761581421, |
|
"rewards/chosen": -2.2767906188964844, |
|
"rewards/margins": 10.406248092651367, |
|
"rewards/rejected": -12.683038711547852, |
|
"step": 320 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"step": 327, |
|
"total_flos": 0.0, |
|
"train_loss": 0.23720380818807385, |
|
"train_runtime": 5120.8034, |
|
"train_samples_per_second": 4.087, |
|
"train_steps_per_second": 0.064 |
|
} |
|
], |
|
"logging_steps": 10, |
|
"max_steps": 327, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 1, |
|
"save_steps": 100, |
|
"total_flos": 0.0, |
|
"train_batch_size": 8, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|