| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.986666666666667, | |
| "eval_steps": 500, | |
| "global_step": 168, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.17777777777777778, | |
| "grad_norm": 7.277172565460205, | |
| "learning_rate": 2.9411764705882355e-06, | |
| "logits/chosen": -0.4413936734199524, | |
| "logits/rejected": -0.5268994569778442, | |
| "logps/chosen": -1.6909767389297485, | |
| "logps/rejected": -2.157106876373291, | |
| "loss": 1.742, | |
| "odds_ratio_loss": 15.103334426879883, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.1690976917743683, | |
| "rewards/margins": 0.046613018959760666, | |
| "rewards/rejected": -0.21571068465709686, | |
| "sft_loss": 0.2316340208053589, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.35555555555555557, | |
| "grad_norm": 6.993782043457031, | |
| "learning_rate": 4.995131923687488e-06, | |
| "logits/chosen": -0.4119906425476074, | |
| "logits/rejected": -0.48687878251075745, | |
| "logps/chosen": -1.74609375, | |
| "logps/rejected": -2.23624849319458, | |
| "loss": 1.7977, | |
| "odds_ratio_loss": 15.667881965637207, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.17460939288139343, | |
| "rewards/margins": 0.04901544004678726, | |
| "rewards/rejected": -0.22362485527992249, | |
| "sft_loss": 0.23094086349010468, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.5333333333333333, | |
| "grad_norm": 7.126066207885742, | |
| "learning_rate": 4.90911473983908e-06, | |
| "logits/chosen": -0.3993254601955414, | |
| "logits/rejected": -0.47226476669311523, | |
| "logps/chosen": -1.6316206455230713, | |
| "logps/rejected": -2.0731987953186035, | |
| "loss": 1.685, | |
| "odds_ratio_loss": 14.636944770812988, | |
| "rewards/accuracies": 0.8062499761581421, | |
| "rewards/chosen": -0.16316203773021698, | |
| "rewards/margins": 0.044157832860946655, | |
| "rewards/rejected": -0.20731988549232483, | |
| "sft_loss": 0.22126968204975128, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.7111111111111111, | |
| "grad_norm": 6.876996040344238, | |
| "learning_rate": 4.71919261421297e-06, | |
| "logits/chosen": -0.4370671212673187, | |
| "logits/rejected": -0.5137215852737427, | |
| "logps/chosen": -1.3095628023147583, | |
| "logps/rejected": -1.8373353481292725, | |
| "loss": 1.3576, | |
| "odds_ratio_loss": 11.971807479858398, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": -0.13095629215240479, | |
| "rewards/margins": 0.052777260541915894, | |
| "rewards/rejected": -0.18373355269432068, | |
| "sft_loss": 0.16043850779533386, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.8888888888888888, | |
| "grad_norm": 3.4186394214630127, | |
| "learning_rate": 4.43355687413747e-06, | |
| "logits/chosen": -0.4473581910133362, | |
| "logits/rejected": -0.5121863484382629, | |
| "logps/chosen": -1.047701120376587, | |
| "logps/rejected": -1.479405164718628, | |
| "loss": 1.0996, | |
| "odds_ratio_loss": 9.729310989379883, | |
| "rewards/accuracies": 0.7562500238418579, | |
| "rewards/chosen": -0.10477012395858765, | |
| "rewards/margins": 0.04317039996385574, | |
| "rewards/rejected": -0.1479405164718628, | |
| "sft_loss": 0.12663736939430237, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 1.0666666666666667, | |
| "grad_norm": 4.834543704986572, | |
| "learning_rate": 4.064526968101844e-06, | |
| "logits/chosen": -0.39888542890548706, | |
| "logits/rejected": -0.4785118103027344, | |
| "logps/chosen": -0.7870318293571472, | |
| "logps/rejected": -1.2147241830825806, | |
| "loss": 0.8362, | |
| "odds_ratio_loss": 7.4531731605529785, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": -0.07870318740606308, | |
| "rewards/margins": 0.04276924580335617, | |
| "rewards/rejected": -0.12147243320941925, | |
| "sft_loss": 0.09090302884578705, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 1.2444444444444445, | |
| "grad_norm": 2.8773512840270996, | |
| "learning_rate": 3.6280191288478437e-06, | |
| "logits/chosen": -0.4217619299888611, | |
| "logits/rejected": -0.49851852655410767, | |
| "logps/chosen": -0.5581613779067993, | |
| "logps/rejected": -1.076547384262085, | |
| "loss": 0.5998, | |
| "odds_ratio_loss": 5.396755695343018, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -0.05581614375114441, | |
| "rewards/margins": 0.051838599145412445, | |
| "rewards/rejected": -0.10765473544597626, | |
| "sft_loss": 0.06012673303484917, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 1.4222222222222223, | |
| "grad_norm": 2.8275744915008545, | |
| "learning_rate": 3.142859907420615e-06, | |
| "logits/chosen": -0.39267396926879883, | |
| "logits/rejected": -0.46761494874954224, | |
| "logps/chosen": -0.43946489691734314, | |
| "logps/rejected": -0.9307826161384583, | |
| "loss": 0.4834, | |
| "odds_ratio_loss": 4.3501482009887695, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": -0.04394649341702461, | |
| "rewards/margins": 0.049131762236356735, | |
| "rewards/rejected": -0.09307824075222015, | |
| "sft_loss": 0.04834695905447006, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 1.6, | |
| "grad_norm": 2.467008590698242, | |
| "learning_rate": 2.629974185404951e-06, | |
| "logits/chosen": -0.3878730535507202, | |
| "logits/rejected": -0.4690762460231781, | |
| "logps/chosen": -0.3482344150543213, | |
| "logps/rejected": -0.9180346727371216, | |
| "loss": 0.3918, | |
| "odds_ratio_loss": 3.5334713459014893, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.03482344001531601, | |
| "rewards/margins": 0.05698002502322197, | |
| "rewards/rejected": -0.09180346131324768, | |
| "sft_loss": 0.03849966078996658, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 1.7777777777777777, | |
| "grad_norm": 2.4158785343170166, | |
| "learning_rate": 2.1114826863194882e-06, | |
| "logits/chosen": -0.3763123154640198, | |
| "logits/rejected": -0.47139158844947815, | |
| "logps/chosen": -0.27210018038749695, | |
| "logps/rejected": -0.8651291728019714, | |
| "loss": 0.3089, | |
| "odds_ratio_loss": 2.8031647205352783, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.027210021391510963, | |
| "rewards/margins": 0.05930290371179581, | |
| "rewards/rejected": -0.08651291579008102, | |
| "sft_loss": 0.02853994630277157, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 1.9555555555555557, | |
| "grad_norm": 2.5559449195861816, | |
| "learning_rate": 1.6097479104361328e-06, | |
| "logits/chosen": -0.4010067880153656, | |
| "logits/rejected": -0.4844478666782379, | |
| "logps/chosen": -0.2540341019630432, | |
| "logps/rejected": -0.6811193823814392, | |
| "loss": 0.3004, | |
| "odds_ratio_loss": 2.749537706375122, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.02540341392159462, | |
| "rewards/margins": 0.04270853474736214, | |
| "rewards/rejected": -0.06811194121837616, | |
| "sft_loss": 0.025406217202544212, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 2.1333333333333333, | |
| "grad_norm": 2.1673660278320312, | |
| "learning_rate": 1.1464096417858821e-06, | |
| "logits/chosen": -0.3880887031555176, | |
| "logits/rejected": -0.47383826971054077, | |
| "logps/chosen": -0.17626990377902985, | |
| "logps/rejected": -0.8207994699478149, | |
| "loss": 0.2064, | |
| "odds_ratio_loss": 1.8402626514434814, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.017626991495490074, | |
| "rewards/margins": 0.06445295363664627, | |
| "rewards/rejected": -0.0820799320936203, | |
| "sft_loss": 0.022340241819620132, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 2.311111111111111, | |
| "grad_norm": 1.6694434881210327, | |
| "learning_rate": 7.414516258630245e-07, | |
| "logits/chosen": -0.4004104733467102, | |
| "logits/rejected": -0.48559218645095825, | |
| "logps/chosen": -0.18084414303302765, | |
| "logps/rejected": -0.7277659177780151, | |
| "loss": 0.2208, | |
| "odds_ratio_loss": 1.993011474609375, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.018084412440657616, | |
| "rewards/margins": 0.054692186415195465, | |
| "rewards/rejected": -0.07277660071849823, | |
| "sft_loss": 0.021496199071407318, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 2.488888888888889, | |
| "grad_norm": 2.6840009689331055, | |
| "learning_rate": 4.123396721497977e-07, | |
| "logits/chosen": -0.38309675455093384, | |
| "logits/rejected": -0.4632611870765686, | |
| "logps/chosen": -0.17778636515140533, | |
| "logps/rejected": -0.7036616802215576, | |
| "loss": 0.2182, | |
| "odds_ratio_loss": 1.9652601480484009, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": -0.017778638750314713, | |
| "rewards/margins": 0.052587538957595825, | |
| "rewards/rejected": -0.07036618143320084, | |
| "sft_loss": 0.021626269444823265, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 2.6666666666666665, | |
| "grad_norm": 2.5022149085998535, | |
| "learning_rate": 1.7326835503629542e-07, | |
| "logits/chosen": -0.41912952065467834, | |
| "logits/rejected": -0.48157674074172974, | |
| "logps/chosen": -0.1846485584974289, | |
| "logps/rejected": -0.6781209707260132, | |
| "loss": 0.2302, | |
| "odds_ratio_loss": 2.0842204093933105, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.01846485398709774, | |
| "rewards/margins": 0.04934724420309067, | |
| "rewards/rejected": -0.06781210005283356, | |
| "sft_loss": 0.02174842543900013, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 2.8444444444444446, | |
| "grad_norm": 2.2835490703582764, | |
| "learning_rate": 3.4548802869627806e-08, | |
| "logits/chosen": -0.3882659673690796, | |
| "logits/rejected": -0.46682611107826233, | |
| "logps/chosen": -0.15616735816001892, | |
| "logps/rejected": -0.6447629332542419, | |
| "loss": 0.193, | |
| "odds_ratio_loss": 1.7490888833999634, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -0.015616737306118011, | |
| "rewards/margins": 0.04885955527424812, | |
| "rewards/rejected": -0.06447629630565643, | |
| "sft_loss": 0.018099095672369003, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 2.986666666666667, | |
| "step": 168, | |
| "total_flos": 2.94774302048256e+16, | |
| "train_loss": 0.703327341448693, | |
| "train_runtime": 380.828, | |
| "train_samples_per_second": 7.09, | |
| "train_steps_per_second": 0.441 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 168, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 2.94774302048256e+16, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |