|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 1.0, |
|
"eval_steps": 100, |
|
"global_step": 329, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"dpo_losses": 0.6931471824645996, |
|
"epoch": 0.0, |
|
"grad_norm": 2.0956511335509704, |
|
"learning_rate": 1.5151515151515152e-08, |
|
"logits/chosen": -2.6820077896118164, |
|
"logits/rejected": -2.6930205821990967, |
|
"logps/chosen": -281.2528381347656, |
|
"logps/rejected": -258.0622253417969, |
|
"loss": 0.6931, |
|
"positive_losses": 0.0, |
|
"rewards/accuracies": 0.0, |
|
"rewards/chosen": 0.0, |
|
"rewards/margins": 0.0, |
|
"rewards/margins_max": 0.0, |
|
"rewards/margins_min": 0.0, |
|
"rewards/margins_std": 0.0, |
|
"rewards/rejected": 0.0, |
|
"step": 1 |
|
}, |
|
{ |
|
"dpo_losses": 0.6928904056549072, |
|
"epoch": 0.03, |
|
"grad_norm": 25.14467901589718, |
|
"learning_rate": 1.5151515151515152e-07, |
|
"logits/chosen": -2.769256353378296, |
|
"logits/rejected": -2.7548859119415283, |
|
"logps/chosen": -284.5684814453125, |
|
"logps/rejected": -249.87168884277344, |
|
"loss": 0.6979, |
|
"positive_losses": 0.05147833377122879, |
|
"rewards/accuracies": 0.4583333432674408, |
|
"rewards/chosen": 0.0003567033272702247, |
|
"rewards/margins": 0.000515056774020195, |
|
"rewards/margins_max": 0.0027906966861337423, |
|
"rewards/margins_min": -0.0017034021439030766, |
|
"rewards/margins_std": 0.0019198498921468854, |
|
"rewards/rejected": -0.00015835335943847895, |
|
"step": 10 |
|
}, |
|
{ |
|
"dpo_losses": 0.6927427053451538, |
|
"epoch": 0.06, |
|
"grad_norm": 15.081930583547756, |
|
"learning_rate": 3.0303030303030305e-07, |
|
"logits/chosen": -2.8346035480499268, |
|
"logits/rejected": -2.7817800045013428, |
|
"logps/chosen": -291.3860168457031, |
|
"logps/rejected": -270.3315124511719, |
|
"loss": 0.6968, |
|
"positive_losses": 0.02039351500570774, |
|
"rewards/accuracies": 0.612500011920929, |
|
"rewards/chosen": 0.001457269536331296, |
|
"rewards/margins": 0.0008110570488497615, |
|
"rewards/margins_max": 0.0036527395714074373, |
|
"rewards/margins_min": -0.0018750065937638283, |
|
"rewards/margins_std": 0.0025213216431438923, |
|
"rewards/rejected": 0.0006462126038968563, |
|
"step": 20 |
|
}, |
|
{ |
|
"dpo_losses": 0.6933337450027466, |
|
"epoch": 0.09, |
|
"grad_norm": 9.990963095062185, |
|
"learning_rate": 4.545454545454545e-07, |
|
"logits/chosen": -2.8629353046417236, |
|
"logits/rejected": -2.8151936531066895, |
|
"logps/chosen": -258.76837158203125, |
|
"logps/rejected": -226.8423614501953, |
|
"loss": 0.6951, |
|
"positive_losses": 0.014158916659653187, |
|
"rewards/accuracies": 0.4375, |
|
"rewards/chosen": 0.004769365303218365, |
|
"rewards/margins": -0.00037118454929441214, |
|
"rewards/margins_max": 0.002330085728317499, |
|
"rewards/margins_min": -0.0032659140415489674, |
|
"rewards/margins_std": 0.0024877875111997128, |
|
"rewards/rejected": 0.005140549503266811, |
|
"step": 30 |
|
}, |
|
{ |
|
"dpo_losses": 0.6926981806755066, |
|
"epoch": 0.12, |
|
"grad_norm": 1.604193261987099, |
|
"learning_rate": 4.993103596812268e-07, |
|
"logits/chosen": -2.829625129699707, |
|
"logits/rejected": -2.764366626739502, |
|
"logps/chosen": -316.5284729003906, |
|
"logps/rejected": -223.76123046875, |
|
"loss": 0.693, |
|
"positive_losses": 0.007878494448959827, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": 0.010137422941625118, |
|
"rewards/margins": 0.0009021027944982052, |
|
"rewards/margins_max": 0.005337115842849016, |
|
"rewards/margins_min": -0.0031714539509266615, |
|
"rewards/margins_std": 0.0037328810431063175, |
|
"rewards/rejected": 0.0092353206127882, |
|
"step": 40 |
|
}, |
|
{ |
|
"dpo_losses": 0.6929700374603271, |
|
"epoch": 0.15, |
|
"grad_norm": 7.523440542250164, |
|
"learning_rate": 4.959416858332709e-07, |
|
"logits/chosen": -2.790869951248169, |
|
"logits/rejected": -2.8046393394470215, |
|
"logps/chosen": -241.8709259033203, |
|
"logps/rejected": -278.8560791015625, |
|
"loss": 0.6937, |
|
"positive_losses": 0.0050521851517260075, |
|
"rewards/accuracies": 0.5375000238418579, |
|
"rewards/chosen": 0.010639366693794727, |
|
"rewards/margins": 0.0003602326032705605, |
|
"rewards/margins_max": 0.005497736390680075, |
|
"rewards/margins_min": -0.004975494462996721, |
|
"rewards/margins_std": 0.0046127657406032085, |
|
"rewards/rejected": 0.010279135778546333, |
|
"step": 50 |
|
}, |
|
{ |
|
"dpo_losses": 0.6924153566360474, |
|
"epoch": 0.18, |
|
"grad_norm": 8.317758374112413, |
|
"learning_rate": 4.898051734555674e-07, |
|
"logits/chosen": -2.834188938140869, |
|
"logits/rejected": -2.8444936275482178, |
|
"logps/chosen": -320.539794921875, |
|
"logps/rejected": -281.9419860839844, |
|
"loss": 0.6928, |
|
"positive_losses": 0.0005203246837481856, |
|
"rewards/accuracies": 0.5375000238418579, |
|
"rewards/chosen": 0.013813069090247154, |
|
"rewards/margins": 0.0014724148204550147, |
|
"rewards/margins_max": 0.008079244755208492, |
|
"rewards/margins_min": -0.003974889405071735, |
|
"rewards/margins_std": 0.005347874015569687, |
|
"rewards/rejected": 0.012340654619038105, |
|
"step": 60 |
|
}, |
|
{ |
|
"dpo_losses": 0.693148672580719, |
|
"epoch": 0.21, |
|
"grad_norm": 7.343753116688168, |
|
"learning_rate": 4.809698831278217e-07, |
|
"logits/chosen": -2.7492079734802246, |
|
"logits/rejected": -2.735715389251709, |
|
"logps/chosen": -265.208251953125, |
|
"logps/rejected": -245.1059112548828, |
|
"loss": 0.6937, |
|
"positive_losses": 0.01350479107350111, |
|
"rewards/accuracies": 0.48750001192092896, |
|
"rewards/chosen": 0.012691942974925041, |
|
"rewards/margins": 6.041547749191523e-06, |
|
"rewards/margins_max": 0.0061768521554768085, |
|
"rewards/margins_min": -0.0060454607009887695, |
|
"rewards/margins_std": 0.00548047199845314, |
|
"rewards/rejected": 0.012685902416706085, |
|
"step": 70 |
|
}, |
|
{ |
|
"dpo_losses": 0.6920666694641113, |
|
"epoch": 0.24, |
|
"grad_norm": 2.0944283368710837, |
|
"learning_rate": 4.6953524759527053e-07, |
|
"logits/chosen": -2.8422322273254395, |
|
"logits/rejected": -2.815117597579956, |
|
"logps/chosen": -280.7608947753906, |
|
"logps/rejected": -273.6212158203125, |
|
"loss": 0.6925, |
|
"positive_losses": 0.00601539621129632, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": 0.01512569934129715, |
|
"rewards/margins": 0.0021733222529292107, |
|
"rewards/margins_max": 0.008934634737670422, |
|
"rewards/margins_min": -0.004246932454407215, |
|
"rewards/margins_std": 0.006057220045477152, |
|
"rewards/rejected": 0.012952374294400215, |
|
"step": 80 |
|
}, |
|
{ |
|
"dpo_losses": 0.6931439638137817, |
|
"epoch": 0.27, |
|
"grad_norm": 7.341115665723385, |
|
"learning_rate": 4.5562995274820283e-07, |
|
"logits/chosen": -2.799257278442383, |
|
"logits/rejected": -2.7461183071136475, |
|
"logps/chosen": -294.0331726074219, |
|
"logps/rejected": -290.0491638183594, |
|
"loss": 0.6927, |
|
"positive_losses": 0.0, |
|
"rewards/accuracies": 0.5249999761581421, |
|
"rewards/chosen": 0.0151880057528615, |
|
"rewards/margins": 1.8423608707962558e-05, |
|
"rewards/margins_max": 0.006872598081827164, |
|
"rewards/margins_min": -0.007915332913398743, |
|
"rewards/margins_std": 0.006555147469043732, |
|
"rewards/rejected": 0.015169581398367882, |
|
"step": 90 |
|
}, |
|
{ |
|
"dpo_losses": 0.6915194988250732, |
|
"epoch": 0.3, |
|
"grad_norm": 1.7021064862560185, |
|
"learning_rate": 4.394104893853007e-07, |
|
"logits/chosen": -2.8969879150390625, |
|
"logits/rejected": -2.8577048778533936, |
|
"logps/chosen": -271.9718017578125, |
|
"logps/rejected": -255.94015502929688, |
|
"loss": 0.6925, |
|
"positive_losses": 0.0063987732864916325, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": 0.01490433793514967, |
|
"rewards/margins": 0.0032706786878407, |
|
"rewards/margins_max": 0.01071108691394329, |
|
"rewards/margins_min": -0.004061442334204912, |
|
"rewards/margins_std": 0.006633488927036524, |
|
"rewards/rejected": 0.011633659712970257, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"eval_dpo_losses": 0.6918572783470154, |
|
"eval_logits/chosen": -2.8058130741119385, |
|
"eval_logits/rejected": -2.7668540477752686, |
|
"eval_logps/chosen": -282.8757629394531, |
|
"eval_logps/rejected": -257.1208801269531, |
|
"eval_loss": 0.6926050782203674, |
|
"eval_positive_losses": 0.002994521986693144, |
|
"eval_rewards/accuracies": 0.6330000162124634, |
|
"eval_rewards/chosen": 0.017176620662212372, |
|
"eval_rewards/margins": 0.0025962977670133114, |
|
"eval_rewards/margins_max": 0.014017846435308456, |
|
"eval_rewards/margins_min": -0.007906544953584671, |
|
"eval_rewards/margins_std": 0.00730311032384634, |
|
"eval_rewards/rejected": 0.014580323360860348, |
|
"eval_runtime": 429.0754, |
|
"eval_samples_per_second": 4.661, |
|
"eval_steps_per_second": 0.291, |
|
"step": 100 |
|
}, |
|
{ |
|
"dpo_losses": 0.6914526224136353, |
|
"epoch": 0.33, |
|
"grad_norm": 2.1403573418786492, |
|
"learning_rate": 4.2105939205932005e-07, |
|
"logits/chosen": -2.764038324356079, |
|
"logits/rejected": -2.7474615573883057, |
|
"logps/chosen": -310.06982421875, |
|
"logps/rejected": -234.0756072998047, |
|
"loss": 0.6917, |
|
"positive_losses": 0.0032207488548010588, |
|
"rewards/accuracies": 0.6499999761581421, |
|
"rewards/chosen": 0.016864223405718803, |
|
"rewards/margins": 0.0034028415102511644, |
|
"rewards/margins_max": 0.010497726500034332, |
|
"rewards/margins_min": -0.0030488395132124424, |
|
"rewards/margins_std": 0.006098336540162563, |
|
"rewards/rejected": 0.013461383990943432, |
|
"step": 110 |
|
}, |
|
{ |
|
"dpo_losses": 0.691475510597229, |
|
"epoch": 0.36, |
|
"grad_norm": 2.00482561166422, |
|
"learning_rate": 4.0078318482522114e-07, |
|
"logits/chosen": -2.752159833908081, |
|
"logits/rejected": -2.750898838043213, |
|
"logps/chosen": -321.77813720703125, |
|
"logps/rejected": -272.95037841796875, |
|
"loss": 0.6922, |
|
"positive_losses": 0.011810302734375, |
|
"rewards/accuracies": 0.6625000238418579, |
|
"rewards/chosen": 0.017863672226667404, |
|
"rewards/margins": 0.003372038947418332, |
|
"rewards/margins_max": 0.014931520447134972, |
|
"rewards/margins_min": -0.006266787648200989, |
|
"rewards/margins_std": 0.009545858018100262, |
|
"rewards/rejected": 0.01449163444340229, |
|
"step": 120 |
|
}, |
|
{ |
|
"dpo_losses": 0.692450225353241, |
|
"epoch": 0.4, |
|
"grad_norm": 1.6365937615424984, |
|
"learning_rate": 3.7881005700938627e-07, |
|
"logits/chosen": -2.8213768005371094, |
|
"logits/rejected": -2.831343650817871, |
|
"logps/chosen": -264.4930114746094, |
|
"logps/rejected": -232.359130859375, |
|
"loss": 0.6926, |
|
"positive_losses": 0.008326339535415173, |
|
"rewards/accuracies": 0.5, |
|
"rewards/chosen": 0.01900574564933777, |
|
"rewards/margins": 0.001420500222593546, |
|
"rewards/margins_max": 0.011069941334426403, |
|
"rewards/margins_min": -0.009186895564198494, |
|
"rewards/margins_std": 0.008960539475083351, |
|
"rewards/rejected": 0.01758524402976036, |
|
"step": 130 |
|
}, |
|
{ |
|
"dpo_losses": 0.690600574016571, |
|
"epoch": 0.43, |
|
"grad_norm": 1.9064212159356546, |
|
"learning_rate": 3.5538729515692354e-07, |
|
"logits/chosen": -2.7800724506378174, |
|
"logits/rejected": -2.763850450515747, |
|
"logps/chosen": -292.2286682128906, |
|
"logps/rejected": -268.72857666015625, |
|
"loss": 0.6912, |
|
"positive_losses": 0.013485336676239967, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": 0.02164594829082489, |
|
"rewards/margins": 0.005139264743775129, |
|
"rewards/margins_max": 0.017013436183333397, |
|
"rewards/margins_min": -0.0060675484128296375, |
|
"rewards/margins_std": 0.010294691659510136, |
|
"rewards/rejected": 0.016506681218743324, |
|
"step": 140 |
|
}, |
|
{ |
|
"dpo_losses": 0.6907236576080322, |
|
"epoch": 0.46, |
|
"grad_norm": 1.4378221399671829, |
|
"learning_rate": 3.3077850005803125e-07, |
|
"logits/chosen": -2.842320203781128, |
|
"logits/rejected": -2.8206865787506104, |
|
"logps/chosen": -268.4656066894531, |
|
"logps/rejected": -243.43276977539062, |
|
"loss": 0.6925, |
|
"positive_losses": 0.0013946533435955644, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": 0.021758494898676872, |
|
"rewards/margins": 0.004882398992776871, |
|
"rewards/margins_max": 0.017139647156000137, |
|
"rewards/margins_min": -0.0063543543219566345, |
|
"rewards/margins_std": 0.010589024983346462, |
|
"rewards/rejected": 0.01687609776854515, |
|
"step": 150 |
|
}, |
|
{ |
|
"dpo_losses": 0.6922906041145325, |
|
"epoch": 0.49, |
|
"grad_norm": 2.1178244445563013, |
|
"learning_rate": 3.0526062017313247e-07, |
|
"logits/chosen": -2.7998666763305664, |
|
"logits/rejected": -2.7823493480682373, |
|
"logps/chosen": -253.3588104248047, |
|
"logps/rejected": -238.626708984375, |
|
"loss": 0.6933, |
|
"positive_losses": 0.0011047363514080644, |
|
"rewards/accuracies": 0.4625000059604645, |
|
"rewards/chosen": 0.021453257650136948, |
|
"rewards/margins": 0.001750586787238717, |
|
"rewards/margins_max": 0.014365032315254211, |
|
"rewards/margins_min": -0.009759698994457722, |
|
"rewards/margins_std": 0.010657011531293392, |
|
"rewards/rejected": 0.019702669233083725, |
|
"step": 160 |
|
}, |
|
{ |
|
"dpo_losses": 0.6903497576713562, |
|
"epoch": 0.52, |
|
"grad_norm": 1.5792742029110545, |
|
"learning_rate": 2.791208348427426e-07, |
|
"logits/chosen": -2.817678451538086, |
|
"logits/rejected": -2.7354776859283447, |
|
"logps/chosen": -301.37139892578125, |
|
"logps/rejected": -271.1383972167969, |
|
"loss": 0.6907, |
|
"positive_losses": 0.0042434693314135075, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": 0.02347148209810257, |
|
"rewards/margins": 0.0056489938870072365, |
|
"rewards/margins_max": 0.017104174941778183, |
|
"rewards/margins_min": -0.0068167769350111485, |
|
"rewards/margins_std": 0.010564757511019707, |
|
"rewards/rejected": 0.01782248727977276, |
|
"step": 170 |
|
}, |
|
{ |
|
"dpo_losses": 0.6911171674728394, |
|
"epoch": 0.55, |
|
"grad_norm": 11.74545654760677, |
|
"learning_rate": 2.526533223585641e-07, |
|
"logits/chosen": -2.8419086933135986, |
|
"logits/rejected": -2.7769436836242676, |
|
"logps/chosen": -253.8654022216797, |
|
"logps/rejected": -227.02719116210938, |
|
"loss": 0.6917, |
|
"positive_losses": 0.0033275603782385588, |
|
"rewards/accuracies": 0.6625000238418579, |
|
"rewards/chosen": 0.02268325164914131, |
|
"rewards/margins": 0.0040952288545668125, |
|
"rewards/margins_max": 0.016258299350738525, |
|
"rewards/margins_min": -0.004967542830854654, |
|
"rewards/margins_std": 0.00928699504584074, |
|
"rewards/rejected": 0.018588021397590637, |
|
"step": 180 |
|
}, |
|
{ |
|
"dpo_losses": 0.6901928186416626, |
|
"epoch": 0.58, |
|
"grad_norm": 9.4811716718965, |
|
"learning_rate": 2.261559492680755e-07, |
|
"logits/chosen": -2.784456729888916, |
|
"logits/rejected": -2.766946792602539, |
|
"logps/chosen": -298.0020446777344, |
|
"logps/rejected": -268.62164306640625, |
|
"loss": 0.6923, |
|
"positive_losses": 0.0034267425071448088, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": 0.025788750499486923, |
|
"rewards/margins": 0.005970729514956474, |
|
"rewards/margins_max": 0.02267562597990036, |
|
"rewards/margins_min": -0.008256749249994755, |
|
"rewards/margins_std": 0.013814790174365044, |
|
"rewards/rejected": 0.01981802098453045, |
|
"step": 190 |
|
}, |
|
{ |
|
"dpo_losses": 0.6899710893630981, |
|
"epoch": 0.61, |
|
"grad_norm": 2.1557416454862444, |
|
"learning_rate": 1.9992691817133024e-07, |
|
"logits/chosen": -2.787891387939453, |
|
"logits/rejected": -2.756988048553467, |
|
"logps/chosen": -279.0212097167969, |
|
"logps/rejected": -286.3407287597656, |
|
"loss": 0.6928, |
|
"positive_losses": 0.02213897742331028, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": 0.025793900713324547, |
|
"rewards/margins": 0.006411595735698938, |
|
"rewards/margins_max": 0.022542249411344528, |
|
"rewards/margins_min": -0.00852292776107788, |
|
"rewards/margins_std": 0.013604698702692986, |
|
"rewards/rejected": 0.019382307305932045, |
|
"step": 200 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"eval_dpo_losses": 0.6903740763664246, |
|
"eval_logits/chosen": -2.8064374923706055, |
|
"eval_logits/rejected": -2.7678143978118896, |
|
"eval_logps/chosen": -282.0021667480469, |
|
"eval_logps/rejected": -256.5478515625, |
|
"eval_loss": 0.6918678879737854, |
|
"eval_positive_losses": 0.008644149638712406, |
|
"eval_rewards/accuracies": 0.6470000147819519, |
|
"eval_rewards/chosen": 0.025912432000041008, |
|
"eval_rewards/margins": 0.005601577460765839, |
|
"eval_rewards/margins_max": 0.02654200792312622, |
|
"eval_rewards/margins_min": -0.01304223295301199, |
|
"eval_rewards/margins_std": 0.013095145113766193, |
|
"eval_rewards/rejected": 0.02031085453927517, |
|
"eval_runtime": 428.4364, |
|
"eval_samples_per_second": 4.668, |
|
"eval_steps_per_second": 0.292, |
|
"step": 200 |
|
}, |
|
{ |
|
"dpo_losses": 0.6895982623100281, |
|
"epoch": 0.64, |
|
"grad_norm": 9.2776467878139, |
|
"learning_rate": 1.742614117358029e-07, |
|
"logits/chosen": -2.8029887676239014, |
|
"logits/rejected": -2.7589237689971924, |
|
"logps/chosen": -302.7414855957031, |
|
"logps/rejected": -286.43438720703125, |
|
"loss": 0.6912, |
|
"positive_losses": 0.013084793463349342, |
|
"rewards/accuracies": 0.6625000238418579, |
|
"rewards/chosen": 0.02575690671801567, |
|
"rewards/margins": 0.007164796348661184, |
|
"rewards/margins_max": 0.023669250309467316, |
|
"rewards/margins_min": -0.007106063421815634, |
|
"rewards/margins_std": 0.014227842912077904, |
|
"rewards/rejected": 0.0185921099036932, |
|
"step": 210 |
|
}, |
|
{ |
|
"dpo_losses": 0.6903053522109985, |
|
"epoch": 0.67, |
|
"grad_norm": 1.955123489265158, |
|
"learning_rate": 1.4944827069769122e-07, |
|
"logits/chosen": -2.8536009788513184, |
|
"logits/rejected": -2.8277313709259033, |
|
"logps/chosen": -310.047607421875, |
|
"logps/rejected": -264.05352783203125, |
|
"loss": 0.6902, |
|
"positive_losses": 0.002166366670280695, |
|
"rewards/accuracies": 0.6625000238418579, |
|
"rewards/chosen": 0.02925548516213894, |
|
"rewards/margins": 0.005735241807997227, |
|
"rewards/margins_max": 0.02009030058979988, |
|
"rewards/margins_min": -0.007059932686388493, |
|
"rewards/margins_std": 0.01263638585805893, |
|
"rewards/rejected": 0.023520244285464287, |
|
"step": 220 |
|
}, |
|
{ |
|
"dpo_losses": 0.6920837163925171, |
|
"epoch": 0.7, |
|
"grad_norm": 8.688484410559708, |
|
"learning_rate": 1.2576674323558928e-07, |
|
"logits/chosen": -2.822300910949707, |
|
"logits/rejected": -2.8432376384735107, |
|
"logps/chosen": -286.09466552734375, |
|
"logps/rejected": -260.4286804199219, |
|
"loss": 0.6922, |
|
"positive_losses": 0.0013641357654705644, |
|
"rewards/accuracies": 0.550000011920929, |
|
"rewards/chosen": 0.025508206337690353, |
|
"rewards/margins": 0.002203943906351924, |
|
"rewards/margins_max": 0.020830942317843437, |
|
"rewards/margins_min": -0.015516755171120167, |
|
"rewards/margins_std": 0.01637136936187744, |
|
"rewards/rejected": 0.02330426312983036, |
|
"step": 230 |
|
}, |
|
{ |
|
"dpo_losses": 0.690080463886261, |
|
"epoch": 0.73, |
|
"grad_norm": 2.0416140906382805, |
|
"learning_rate": 1.0348334229922676e-07, |
|
"logits/chosen": -2.8787901401519775, |
|
"logits/rejected": -2.831031322479248, |
|
"logps/chosen": -288.3709411621094, |
|
"logps/rejected": -273.1520080566406, |
|
"loss": 0.6918, |
|
"positive_losses": 0.01643219031393528, |
|
"rewards/accuracies": 0.6875, |
|
"rewards/chosen": 0.02646791562438011, |
|
"rewards/margins": 0.00618840241804719, |
|
"rewards/margins_max": 0.022397857159376144, |
|
"rewards/margins_min": -0.006807761732488871, |
|
"rewards/margins_std": 0.013137358240783215, |
|
"rewards/rejected": 0.02027951553463936, |
|
"step": 240 |
|
}, |
|
{ |
|
"dpo_losses": 0.6899579763412476, |
|
"epoch": 0.76, |
|
"grad_norm": 1.9842983441302535, |
|
"learning_rate": 8.284884626103164e-08, |
|
"logits/chosen": -2.8189456462860107, |
|
"logits/rejected": -2.787537097930908, |
|
"logps/chosen": -298.2633056640625, |
|
"logps/rejected": -302.3707580566406, |
|
"loss": 0.6937, |
|
"positive_losses": 0.0013935088645666838, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": 0.02826567366719246, |
|
"rewards/margins": 0.006455945782363415, |
|
"rewards/margins_max": 0.022190529853105545, |
|
"rewards/margins_min": -0.010183097794651985, |
|
"rewards/margins_std": 0.014948679134249687, |
|
"rewards/rejected": 0.02180972695350647, |
|
"step": 250 |
|
}, |
|
{ |
|
"dpo_losses": 0.6888389587402344, |
|
"epoch": 0.79, |
|
"grad_norm": 2.1130940747529685, |
|
"learning_rate": 6.409547664531733e-08, |
|
"logits/chosen": -2.8461780548095703, |
|
"logits/rejected": -2.8126139640808105, |
|
"logps/chosen": -330.7644958496094, |
|
"logps/rejected": -310.1560974121094, |
|
"loss": 0.6907, |
|
"positive_losses": 0.0, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": 0.03217402845621109, |
|
"rewards/margins": 0.008677511475980282, |
|
"rewards/margins_max": 0.021433433517813683, |
|
"rewards/margins_min": -0.0022177654318511486, |
|
"rewards/margins_std": 0.010701724328100681, |
|
"rewards/rejected": 0.023496516048908234, |
|
"step": 260 |
|
}, |
|
{ |
|
"dpo_losses": 0.6895651817321777, |
|
"epoch": 0.82, |
|
"grad_norm": 1.9914333935784974, |
|
"learning_rate": 4.743428469705335e-08, |
|
"logits/chosen": -2.7977194786071777, |
|
"logits/rejected": -2.792222499847412, |
|
"logps/chosen": -301.0746154785156, |
|
"logps/rejected": -305.9708557128906, |
|
"loss": 0.6919, |
|
"positive_losses": 0.020720671862363815, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": 0.027332518249750137, |
|
"rewards/margins": 0.007225664798170328, |
|
"rewards/margins_max": 0.021714068949222565, |
|
"rewards/margins_min": -0.0065118856728076935, |
|
"rewards/margins_std": 0.012981800362467766, |
|
"rewards/rejected": 0.020106855779886246, |
|
"step": 270 |
|
}, |
|
{ |
|
"dpo_losses": 0.6888093948364258, |
|
"epoch": 0.85, |
|
"grad_norm": 2.1081832344794846, |
|
"learning_rate": 3.305277620188826e-08, |
|
"logits/chosen": -2.8479342460632324, |
|
"logits/rejected": -2.829345226287842, |
|
"logps/chosen": -322.39654541015625, |
|
"logps/rejected": -267.4809875488281, |
|
"loss": 0.6907, |
|
"positive_losses": 0.00835342425853014, |
|
"rewards/accuracies": 0.7124999761581421, |
|
"rewards/chosen": 0.03169160336256027, |
|
"rewards/margins": 0.008761906065046787, |
|
"rewards/margins_max": 0.025315579026937485, |
|
"rewards/margins_min": -0.00724324444308877, |
|
"rewards/margins_std": 0.014560697600245476, |
|
"rewards/rejected": 0.02292969450354576, |
|
"step": 280 |
|
}, |
|
{ |
|
"dpo_losses": 0.690091609954834, |
|
"epoch": 0.88, |
|
"grad_norm": 1.7109015474069318, |
|
"learning_rate": 2.1112801287806375e-08, |
|
"logits/chosen": -2.7887511253356934, |
|
"logits/rejected": -2.753249168395996, |
|
"logps/chosen": -271.52618408203125, |
|
"logps/rejected": -243.4734344482422, |
|
"loss": 0.691, |
|
"positive_losses": 0.0007474899175576866, |
|
"rewards/accuracies": 0.6875, |
|
"rewards/chosen": 0.02663729526102543, |
|
"rewards/margins": 0.0061707692220807076, |
|
"rewards/margins_max": 0.020485369488596916, |
|
"rewards/margins_min": -0.006647027097642422, |
|
"rewards/margins_std": 0.012255651876330376, |
|
"rewards/rejected": 0.020466525107622147, |
|
"step": 290 |
|
}, |
|
{ |
|
"dpo_losses": 0.6898671984672546, |
|
"epoch": 0.91, |
|
"grad_norm": 6.914969485002492, |
|
"learning_rate": 1.1748732956682023e-08, |
|
"logits/chosen": -2.8814024925231934, |
|
"logits/rejected": -2.8128414154052734, |
|
"logps/chosen": -321.03570556640625, |
|
"logps/rejected": -283.5750732421875, |
|
"loss": 0.6916, |
|
"positive_losses": 0.020281601697206497, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": 0.02677486464381218, |
|
"rewards/margins": 0.006634272634983063, |
|
"rewards/margins_max": 0.02378956601023674, |
|
"rewards/margins_min": -0.009901603683829308, |
|
"rewards/margins_std": 0.015127221122384071, |
|
"rewards/rejected": 0.020140592008829117, |
|
"step": 300 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"eval_dpo_losses": 0.6899807453155518, |
|
"eval_logits/chosen": -2.8024394512176514, |
|
"eval_logits/rejected": -2.763456344604492, |
|
"eval_logps/chosen": -281.7664489746094, |
|
"eval_logps/rejected": -256.39239501953125, |
|
"eval_loss": 0.6914138197898865, |
|
"eval_positive_losses": 0.006967452820390463, |
|
"eval_rewards/accuracies": 0.6629999876022339, |
|
"eval_rewards/chosen": 0.028269650414586067, |
|
"eval_rewards/margins": 0.0064042662270367146, |
|
"eval_rewards/margins_max": 0.030468182638287544, |
|
"eval_rewards/margins_min": -0.014824692159891129, |
|
"eval_rewards/margins_std": 0.014939413405954838, |
|
"eval_rewards/rejected": 0.02186538279056549, |
|
"eval_runtime": 428.1048, |
|
"eval_samples_per_second": 4.672, |
|
"eval_steps_per_second": 0.292, |
|
"step": 300 |
|
}, |
|
{ |
|
"dpo_losses": 0.690555214881897, |
|
"epoch": 0.94, |
|
"grad_norm": 1.5197312877950733, |
|
"learning_rate": 5.065954844616721e-09, |
|
"logits/chosen": -2.827882766723633, |
|
"logits/rejected": -2.781325340270996, |
|
"logps/chosen": -274.24688720703125, |
|
"logps/rejected": -278.71307373046875, |
|
"loss": 0.6911, |
|
"positive_losses": 0.0, |
|
"rewards/accuracies": 0.612500011920929, |
|
"rewards/chosen": 0.028748828917741776, |
|
"rewards/margins": 0.005246045999228954, |
|
"rewards/margins_max": 0.022478515282273293, |
|
"rewards/margins_min": -0.010068441741168499, |
|
"rewards/margins_std": 0.014408007264137268, |
|
"rewards/rejected": 0.023502781987190247, |
|
"step": 310 |
|
}, |
|
{ |
|
"dpo_losses": 0.6897249817848206, |
|
"epoch": 0.97, |
|
"grad_norm": 1.9024323554426086, |
|
"learning_rate": 1.1396752298723499e-09, |
|
"logits/chosen": -2.869424819946289, |
|
"logits/rejected": -2.8175909519195557, |
|
"logps/chosen": -246.2608642578125, |
|
"logps/rejected": -255.5459442138672, |
|
"loss": 0.6898, |
|
"positive_losses": 0.0, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": 0.026843314990401268, |
|
"rewards/margins": 0.006898718420416117, |
|
"rewards/margins_max": 0.020682070404291153, |
|
"rewards/margins_min": -0.0074739838019013405, |
|
"rewards/margins_std": 0.012545767240226269, |
|
"rewards/rejected": 0.01994459703564644, |
|
"step": 320 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"step": 329, |
|
"total_flos": 0.0, |
|
"train_loss": 0.6924228537771114, |
|
"train_runtime": 3896.0637, |
|
"train_samples_per_second": 1.351, |
|
"train_steps_per_second": 0.084 |
|
} |
|
], |
|
"logging_steps": 10, |
|
"max_steps": 329, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 1, |
|
"save_steps": 100, |
|
"total_flos": 0.0, |
|
"train_batch_size": 4, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|