| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 478, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "debug/losses": 0.36003828048706055, | |
| "debug/policy_weights": 0.5194254517555237, | |
| "debug/raw_losses": 0.6931471824645996, | |
| "epoch": 0.0, | |
| "grad_norm": 5.3582401111914875, | |
| "learning_rate": 1.0416666666666666e-08, | |
| "logits/chosen": -2.888590097427368, | |
| "logits/rejected": -2.871957540512085, | |
| "logps/chosen": -261.88616943359375, | |
| "logps/rejected": -355.31793212890625, | |
| "loss": 0.3648, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "debug/losses": 0.3548344671726227, | |
| "debug/policy_weights": 0.5118452906608582, | |
| "debug/raw_losses": 0.6932389736175537, | |
| "epoch": 0.02, | |
| "grad_norm": 5.014402110225361, | |
| "learning_rate": 1.0416666666666667e-07, | |
| "logits/chosen": -2.7435038089752197, | |
| "logits/rejected": -2.7258925437927246, | |
| "logps/chosen": -256.5353088378906, | |
| "logps/rejected": -244.49546813964844, | |
| "loss": 0.3619, | |
| "rewards/accuracies": 0.4236111044883728, | |
| "rewards/chosen": -8.211319800466299e-05, | |
| "rewards/margins": -0.00017834355821833014, | |
| "rewards/rejected": 9.623030200600624e-05, | |
| "step": 10 | |
| }, | |
| { | |
| "debug/losses": 0.3572663962841034, | |
| "debug/policy_weights": 0.5156484842300415, | |
| "debug/raw_losses": 0.6928108930587769, | |
| "epoch": 0.04, | |
| "grad_norm": 4.715098712970233, | |
| "learning_rate": 2.0833333333333333e-07, | |
| "logits/chosen": -2.8007731437683105, | |
| "logits/rejected": -2.7729618549346924, | |
| "logps/chosen": -271.37860107421875, | |
| "logps/rejected": -288.34271240234375, | |
| "loss": 0.3584, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": 0.0006374852382577956, | |
| "rewards/margins": 0.0006833201623521745, | |
| "rewards/rejected": -4.583501868182793e-05, | |
| "step": 20 | |
| }, | |
| { | |
| "debug/losses": 0.35951218008995056, | |
| "debug/policy_weights": 0.521370530128479, | |
| "debug/raw_losses": 0.6895479559898376, | |
| "epoch": 0.06, | |
| "grad_norm": 4.7982052271779825, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logits/chosen": -2.810387134552002, | |
| "logits/rejected": -2.773030996322632, | |
| "logps/chosen": -291.4071350097656, | |
| "logps/rejected": -258.0267639160156, | |
| "loss": 0.3579, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": 0.004485988523811102, | |
| "rewards/margins": 0.00730940094217658, | |
| "rewards/rejected": -0.002823411952704191, | |
| "step": 30 | |
| }, | |
| { | |
| "debug/losses": 0.35061347484588623, | |
| "debug/policy_weights": 0.5189496278762817, | |
| "debug/raw_losses": 0.6759483218193054, | |
| "epoch": 0.08, | |
| "grad_norm": 4.841334528975048, | |
| "learning_rate": 4.1666666666666667e-07, | |
| "logits/chosen": -2.7438368797302246, | |
| "logits/rejected": -2.7376155853271484, | |
| "logps/chosen": -252.50436401367188, | |
| "logps/rejected": -240.8105010986328, | |
| "loss": 0.3573, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": 0.015405787155032158, | |
| "rewards/margins": 0.03552238270640373, | |
| "rewards/rejected": -0.020116599276661873, | |
| "step": 40 | |
| }, | |
| { | |
| "debug/losses": 0.34173911809921265, | |
| "debug/policy_weights": 0.5125550627708435, | |
| "debug/raw_losses": 0.666517436504364, | |
| "epoch": 0.1, | |
| "grad_norm": 5.135483442753442, | |
| "learning_rate": 4.999733114418725e-07, | |
| "logits/chosen": -2.6830456256866455, | |
| "logits/rejected": -2.6514101028442383, | |
| "logps/chosen": -269.3827819824219, | |
| "logps/rejected": -287.2772216796875, | |
| "loss": 0.3437, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": 0.0019381232559680939, | |
| "rewards/margins": 0.05897800251841545, | |
| "rewards/rejected": -0.05703987926244736, | |
| "step": 50 | |
| }, | |
| { | |
| "debug/losses": 0.3377828598022461, | |
| "debug/policy_weights": 0.5224286317825317, | |
| "debug/raw_losses": 0.6448591947555542, | |
| "epoch": 0.13, | |
| "grad_norm": 5.5606740216520905, | |
| "learning_rate": 4.990398100856366e-07, | |
| "logits/chosen": -2.745028018951416, | |
| "logits/rejected": -2.713804244995117, | |
| "logps/chosen": -266.05279541015625, | |
| "logps/rejected": -305.28271484375, | |
| "loss": 0.3305, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.02193317376077175, | |
| "rewards/margins": 0.1177050918340683, | |
| "rewards/rejected": -0.1396382749080658, | |
| "step": 60 | |
| }, | |
| { | |
| "debug/losses": 0.28029459714889526, | |
| "debug/policy_weights": 0.47101029753685, | |
| "debug/raw_losses": 0.6030661463737488, | |
| "epoch": 0.15, | |
| "grad_norm": 6.284287160526739, | |
| "learning_rate": 4.967775735898179e-07, | |
| "logits/chosen": -2.7072081565856934, | |
| "logits/rejected": -2.7113521099090576, | |
| "logps/chosen": -272.530029296875, | |
| "logps/rejected": -289.87713623046875, | |
| "loss": 0.2826, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.1557193100452423, | |
| "rewards/margins": 0.24685576558113098, | |
| "rewards/rejected": -0.4025750756263733, | |
| "step": 70 | |
| }, | |
| { | |
| "debug/losses": 0.26233458518981934, | |
| "debug/policy_weights": 0.42553257942199707, | |
| "debug/raw_losses": 0.6218484044075012, | |
| "epoch": 0.17, | |
| "grad_norm": 6.8893773770621305, | |
| "learning_rate": 4.931986719649298e-07, | |
| "logits/chosen": -2.800093412399292, | |
| "logits/rejected": -2.7768216133117676, | |
| "logps/chosen": -360.3621520996094, | |
| "logps/rejected": -329.7616882324219, | |
| "loss": 0.2516, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.45188745856285095, | |
| "rewards/margins": 0.26955321431159973, | |
| "rewards/rejected": -0.7214406728744507, | |
| "step": 80 | |
| }, | |
| { | |
| "debug/losses": 0.18998686969280243, | |
| "debug/policy_weights": 0.34327536821365356, | |
| "debug/raw_losses": 0.5527450442314148, | |
| "epoch": 0.19, | |
| "grad_norm": 6.5870040213670435, | |
| "learning_rate": 4.883222001996351e-07, | |
| "logits/chosen": -2.724050998687744, | |
| "logits/rejected": -2.690390110015869, | |
| "logps/chosen": -303.0259704589844, | |
| "logps/rejected": -344.49505615234375, | |
| "loss": 0.2198, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": -0.5087189078330994, | |
| "rewards/margins": 0.4743797779083252, | |
| "rewards/rejected": -0.9830986857414246, | |
| "step": 90 | |
| }, | |
| { | |
| "debug/losses": 0.16577747464179993, | |
| "debug/policy_weights": 0.29914408922195435, | |
| "debug/raw_losses": 0.5552432537078857, | |
| "epoch": 0.21, | |
| "grad_norm": 7.249323730073561, | |
| "learning_rate": 4.821741763807186e-07, | |
| "logits/chosen": -2.6653432846069336, | |
| "logits/rejected": -2.658141613006592, | |
| "logps/chosen": -325.9325866699219, | |
| "logps/rejected": -375.12127685546875, | |
| "loss": 0.1781, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.7297804951667786, | |
| "rewards/margins": 0.511761486530304, | |
| "rewards/rejected": -1.2415419816970825, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "eval_debug/losses": 0.19525206089019775, | |
| "eval_debug/policy_weights": 0.3511899709701538, | |
| "eval_debug/raw_losses": 0.5590351819992065, | |
| "eval_logits/chosen": -2.70804500579834, | |
| "eval_logits/rejected": -2.6910133361816406, | |
| "eval_logps/chosen": -319.9805603027344, | |
| "eval_logps/rejected": -373.18670654296875, | |
| "eval_loss": 0.20066915452480316, | |
| "eval_rewards/accuracies": 0.734375, | |
| "eval_rewards/chosen": -0.6478448510169983, | |
| "eval_rewards/margins": 0.5214487910270691, | |
| "eval_rewards/rejected": -1.1692936420440674, | |
| "eval_runtime": 43.2006, | |
| "eval_samples_per_second": 46.296, | |
| "eval_steps_per_second": 0.741, | |
| "step": 100 | |
| }, | |
| { | |
| "debug/losses": 0.1987285315990448, | |
| "debug/policy_weights": 0.34678488969802856, | |
| "debug/raw_losses": 0.5721240043640137, | |
| "epoch": 0.23, | |
| "grad_norm": 6.9123559213202554, | |
| "learning_rate": 4.747874028753375e-07, | |
| "logits/chosen": -2.7045469284057617, | |
| "logits/rejected": -2.672999858856201, | |
| "logps/chosen": -369.9187927246094, | |
| "logps/rejected": -373.6562194824219, | |
| "loss": 0.1986, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.7313551902770996, | |
| "rewards/margins": 0.47859472036361694, | |
| "rewards/rejected": -1.2099498510360718, | |
| "step": 110 | |
| }, | |
| { | |
| "debug/losses": 0.18334314227104187, | |
| "debug/policy_weights": 0.3257436156272888, | |
| "debug/raw_losses": 0.5537685751914978, | |
| "epoch": 0.25, | |
| "grad_norm": 7.316158832496888, | |
| "learning_rate": 4.662012913161997e-07, | |
| "logits/chosen": -2.6026859283447266, | |
| "logits/rejected": -2.600971221923828, | |
| "logps/chosen": -346.50555419921875, | |
| "logps/rejected": -379.53887939453125, | |
| "loss": 0.1846, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -0.7285534739494324, | |
| "rewards/margins": 0.5237767100334167, | |
| "rewards/rejected": -1.2523301839828491, | |
| "step": 120 | |
| }, | |
| { | |
| "debug/losses": 0.17840833961963654, | |
| "debug/policy_weights": 0.3271704614162445, | |
| "debug/raw_losses": 0.5529276132583618, | |
| "epoch": 0.27, | |
| "grad_norm": 6.11630399118371, | |
| "learning_rate": 4.5646165232345103e-07, | |
| "logits/chosen": -2.5919995307922363, | |
| "logits/rejected": -2.598027467727661, | |
| "logps/chosen": -346.44610595703125, | |
| "logps/rejected": -391.03778076171875, | |
| "loss": 0.1758, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -0.7799400687217712, | |
| "rewards/margins": 0.5067718029022217, | |
| "rewards/rejected": -1.2867119312286377, | |
| "step": 130 | |
| }, | |
| { | |
| "debug/losses": 0.18315494060516357, | |
| "debug/policy_weights": 0.3452090620994568, | |
| "debug/raw_losses": 0.56131511926651, | |
| "epoch": 0.29, | |
| "grad_norm": 6.845717287191781, | |
| "learning_rate": 4.456204510851956e-07, | |
| "logits/chosen": -2.572934627532959, | |
| "logits/rejected": -2.5713508129119873, | |
| "logps/chosen": -390.03253173828125, | |
| "logps/rejected": -440.16461181640625, | |
| "loss": 0.1668, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.8578819036483765, | |
| "rewards/margins": 0.5912094712257385, | |
| "rewards/rejected": -1.4490914344787598, | |
| "step": 140 | |
| }, | |
| { | |
| "debug/losses": 0.19362398982048035, | |
| "debug/policy_weights": 0.33687275648117065, | |
| "debug/raw_losses": 0.559166669845581, | |
| "epoch": 0.31, | |
| "grad_norm": 7.029648911332738, | |
| "learning_rate": 4.337355301007335e-07, | |
| "logits/chosen": -2.512932300567627, | |
| "logits/rejected": -2.4797143936157227, | |
| "logps/chosen": -357.82464599609375, | |
| "logps/rejected": -402.2618713378906, | |
| "loss": 0.1771, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -0.8432974815368652, | |
| "rewards/margins": 0.48099932074546814, | |
| "rewards/rejected": -1.3242968320846558, | |
| "step": 150 | |
| }, | |
| { | |
| "debug/losses": 0.14652669429779053, | |
| "debug/policy_weights": 0.28093475103378296, | |
| "debug/raw_losses": 0.5318498611450195, | |
| "epoch": 0.33, | |
| "grad_norm": 6.358610241024355, | |
| "learning_rate": 4.2087030056579986e-07, | |
| "logits/chosen": -2.464831590652466, | |
| "logits/rejected": -2.4293084144592285, | |
| "logps/chosen": -370.14971923828125, | |
| "logps/rejected": -408.4079284667969, | |
| "loss": 0.1583, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.0606319904327393, | |
| "rewards/margins": 0.596233606338501, | |
| "rewards/rejected": -1.6568657159805298, | |
| "step": 160 | |
| }, | |
| { | |
| "debug/losses": 0.1372167319059372, | |
| "debug/policy_weights": 0.2504947781562805, | |
| "debug/raw_losses": 0.6000548005104065, | |
| "epoch": 0.36, | |
| "grad_norm": 7.75801911972225, | |
| "learning_rate": 4.070934040463998e-07, | |
| "logits/chosen": -2.361786365509033, | |
| "logits/rejected": -2.332592010498047, | |
| "logps/chosen": -389.86767578125, | |
| "logps/rejected": -423.97698974609375, | |
| "loss": 0.1417, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.4737131595611572, | |
| "rewards/margins": 0.511743426322937, | |
| "rewards/rejected": -1.9854564666748047, | |
| "step": 170 | |
| }, | |
| { | |
| "debug/losses": 0.11748126894235611, | |
| "debug/policy_weights": 0.2296670377254486, | |
| "debug/raw_losses": 0.5410301089286804, | |
| "epoch": 0.38, | |
| "grad_norm": 6.142786182484198, | |
| "learning_rate": 3.9247834624635404e-07, | |
| "logits/chosen": -2.3254575729370117, | |
| "logits/rejected": -2.3115153312683105, | |
| "logps/chosen": -391.5261535644531, | |
| "logps/rejected": -429.21875, | |
| "loss": 0.1327, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -1.446401834487915, | |
| "rewards/margins": 0.6310542821884155, | |
| "rewards/rejected": -2.07745623588562, | |
| "step": 180 | |
| }, | |
| { | |
| "debug/losses": 0.16298121213912964, | |
| "debug/policy_weights": 0.2831325829029083, | |
| "debug/raw_losses": 0.5709558725357056, | |
| "epoch": 0.4, | |
| "grad_norm": 6.941055961208667, | |
| "learning_rate": 3.7710310482256523e-07, | |
| "logits/chosen": -2.378763198852539, | |
| "logits/rejected": -2.3435521125793457, | |
| "logps/chosen": -381.296630859375, | |
| "logps/rejected": -435.13494873046875, | |
| "loss": 0.1447, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.178696870803833, | |
| "rewards/margins": 0.6175613403320312, | |
| "rewards/rejected": -1.7962583303451538, | |
| "step": 190 | |
| }, | |
| { | |
| "debug/losses": 0.1531984657049179, | |
| "debug/policy_weights": 0.27346447110176086, | |
| "debug/raw_losses": 0.5578541159629822, | |
| "epoch": 0.42, | |
| "grad_norm": 9.54639374222291, | |
| "learning_rate": 3.610497133404795e-07, | |
| "logits/chosen": -2.3584351539611816, | |
| "logits/rejected": -2.362753391265869, | |
| "logps/chosen": -359.6307067871094, | |
| "logps/rejected": -422.2249450683594, | |
| "loss": 0.1616, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -1.0809690952301025, | |
| "rewards/margins": 0.6879221796989441, | |
| "rewards/rejected": -1.7688913345336914, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "eval_debug/losses": 0.16714033484458923, | |
| "eval_debug/policy_weights": 0.3173619508743286, | |
| "eval_debug/raw_losses": 0.5079437494277954, | |
| "eval_logits/chosen": -2.44987154006958, | |
| "eval_logits/rejected": -2.4276504516601562, | |
| "eval_logps/chosen": -343.49139404296875, | |
| "eval_logps/rejected": -416.28436279296875, | |
| "eval_loss": 0.16694626212120056, | |
| "eval_rewards/accuracies": 0.7109375, | |
| "eval_rewards/chosen": -0.8829529881477356, | |
| "eval_rewards/margins": 0.7173169851303101, | |
| "eval_rewards/rejected": -1.6002700328826904, | |
| "eval_runtime": 43.2126, | |
| "eval_samples_per_second": 46.283, | |
| "eval_steps_per_second": 0.741, | |
| "step": 200 | |
| }, | |
| { | |
| "debug/losses": 0.14734311401844025, | |
| "debug/policy_weights": 0.2870820164680481, | |
| "debug/raw_losses": 0.5041711330413818, | |
| "epoch": 0.44, | |
| "grad_norm": 7.104709810051969, | |
| "learning_rate": 3.4440382358952115e-07, | |
| "logits/chosen": -2.3574612140655518, | |
| "logits/rejected": -2.326927900314331, | |
| "logps/chosen": -381.2717590332031, | |
| "logps/rejected": -413.890869140625, | |
| "loss": 0.1636, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.9982665777206421, | |
| "rewards/margins": 0.6929588317871094, | |
| "rewards/rejected": -1.6912254095077515, | |
| "step": 210 | |
| }, | |
| { | |
| "debug/losses": 0.13881739974021912, | |
| "debug/policy_weights": 0.27268368005752563, | |
| "debug/raw_losses": 0.5210752487182617, | |
| "epoch": 0.46, | |
| "grad_norm": 8.340698654196919, | |
| "learning_rate": 3.272542485937368e-07, | |
| "logits/chosen": -2.3073763847351074, | |
| "logits/rejected": -2.288245677947998, | |
| "logps/chosen": -370.5925598144531, | |
| "logps/rejected": -423.53021240234375, | |
| "loss": 0.1549, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.1713535785675049, | |
| "rewards/margins": 0.7361623644828796, | |
| "rewards/rejected": -1.9075158834457397, | |
| "step": 220 | |
| }, | |
| { | |
| "debug/losses": 0.15816743671894073, | |
| "debug/policy_weights": 0.2902621626853943, | |
| "debug/raw_losses": 0.509421706199646, | |
| "epoch": 0.48, | |
| "grad_norm": 7.2612463171873225, | |
| "learning_rate": 3.096924887558854e-07, | |
| "logits/chosen": -2.339087724685669, | |
| "logits/rejected": -2.31463360786438, | |
| "logps/chosen": -342.2999572753906, | |
| "logps/rejected": -425.86181640625, | |
| "loss": 0.169, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -1.0157617330551147, | |
| "rewards/margins": 0.8210276365280151, | |
| "rewards/rejected": -1.8367893695831299, | |
| "step": 230 | |
| }, | |
| { | |
| "debug/losses": 0.15576457977294922, | |
| "debug/policy_weights": 0.29614073038101196, | |
| "debug/raw_losses": 0.4828997552394867, | |
| "epoch": 0.5, | |
| "grad_norm": 7.345094511114747, | |
| "learning_rate": 2.9181224366319943e-07, | |
| "logits/chosen": -2.3611202239990234, | |
| "logits/rejected": -2.317119836807251, | |
| "logps/chosen": -368.0779113769531, | |
| "logps/rejected": -434.31549072265625, | |
| "loss": 0.1514, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": -1.1054710149765015, | |
| "rewards/margins": 0.8346185684204102, | |
| "rewards/rejected": -1.9400895833969116, | |
| "step": 240 | |
| }, | |
| { | |
| "debug/losses": 0.12131881713867188, | |
| "debug/policy_weights": 0.2461313009262085, | |
| "debug/raw_losses": 0.46923789381980896, | |
| "epoch": 0.52, | |
| "grad_norm": 6.902697741869489, | |
| "learning_rate": 2.7370891215954565e-07, | |
| "logits/chosen": -2.2654645442962646, | |
| "logits/rejected": -2.2106964588165283, | |
| "logps/chosen": -434.3672790527344, | |
| "logps/rejected": -491.00750732421875, | |
| "loss": 0.1285, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -1.4400012493133545, | |
| "rewards/margins": 0.9503369331359863, | |
| "rewards/rejected": -2.390338182449341, | |
| "step": 250 | |
| }, | |
| { | |
| "debug/losses": 0.1332876831293106, | |
| "debug/policy_weights": 0.27705293893814087, | |
| "debug/raw_losses": 0.5086624622344971, | |
| "epoch": 0.54, | |
| "grad_norm": 7.179715362550834, | |
| "learning_rate": 2.55479083351317e-07, | |
| "logits/chosen": -2.316641330718994, | |
| "logits/rejected": -2.3014323711395264, | |
| "logps/chosen": -410.114990234375, | |
| "logps/rejected": -454.8689880371094, | |
| "loss": 0.1406, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -1.2414360046386719, | |
| "rewards/margins": 0.8403455018997192, | |
| "rewards/rejected": -2.0817813873291016, | |
| "step": 260 | |
| }, | |
| { | |
| "debug/losses": 0.168529212474823, | |
| "debug/policy_weights": 0.2988266348838806, | |
| "debug/raw_losses": 0.5682334899902344, | |
| "epoch": 0.56, | |
| "grad_norm": 7.216808591858915, | |
| "learning_rate": 2.3722002126275822e-07, | |
| "logits/chosen": -2.3377718925476074, | |
| "logits/rejected": -2.3152408599853516, | |
| "logps/chosen": -379.34661865234375, | |
| "logps/rejected": -430.7207946777344, | |
| "loss": 0.1519, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.0926826000213623, | |
| "rewards/margins": 0.6951464414596558, | |
| "rewards/rejected": -1.7878293991088867, | |
| "step": 270 | |
| }, | |
| { | |
| "debug/losses": 0.15156733989715576, | |
| "debug/policy_weights": 0.2692407965660095, | |
| "debug/raw_losses": 0.5413981080055237, | |
| "epoch": 0.59, | |
| "grad_norm": 9.464551455783996, | |
| "learning_rate": 2.19029145890313e-07, | |
| "logits/chosen": -2.295574426651001, | |
| "logits/rejected": -2.2587995529174805, | |
| "logps/chosen": -383.4193420410156, | |
| "logps/rejected": -444.91680908203125, | |
| "loss": 0.1498, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -1.302856683731079, | |
| "rewards/margins": 0.7735105156898499, | |
| "rewards/rejected": -2.076367139816284, | |
| "step": 280 | |
| }, | |
| { | |
| "debug/losses": 0.14380133152008057, | |
| "debug/policy_weights": 0.26454827189445496, | |
| "debug/raw_losses": 0.5405210256576538, | |
| "epoch": 0.61, | |
| "grad_norm": 7.542791313020671, | |
| "learning_rate": 2.0100351342479216e-07, | |
| "logits/chosen": -2.3062846660614014, | |
| "logits/rejected": -2.2651028633117676, | |
| "logps/chosen": -386.890625, | |
| "logps/rejected": -450.9478454589844, | |
| "loss": 0.1352, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -1.4489481449127197, | |
| "rewards/margins": 0.7288294434547424, | |
| "rewards/rejected": -2.1777775287628174, | |
| "step": 290 | |
| }, | |
| { | |
| "debug/losses": 0.14622345566749573, | |
| "debug/policy_weights": 0.2636463940143585, | |
| "debug/raw_losses": 0.536266028881073, | |
| "epoch": 0.63, | |
| "grad_norm": 7.687631612230176, | |
| "learning_rate": 1.8323929841460178e-07, | |
| "logits/chosen": -2.2653095722198486, | |
| "logits/rejected": -2.2190821170806885, | |
| "logps/chosen": -444.52960205078125, | |
| "logps/rejected": -489.06817626953125, | |
| "loss": 0.1343, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.5472103357315063, | |
| "rewards/margins": 0.8187928199768066, | |
| "rewards/rejected": -2.3660032749176025, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "eval_debug/losses": 0.13651318848133087, | |
| "eval_debug/policy_weights": 0.26656585931777954, | |
| "eval_debug/raw_losses": 0.49531808495521545, | |
| "eval_logits/chosen": -2.2618136405944824, | |
| "eval_logits/rejected": -2.228300094604492, | |
| "eval_logps/chosen": -405.40423583984375, | |
| "eval_logps/rejected": -493.411376953125, | |
| "eval_loss": 0.13676287233829498, | |
| "eval_rewards/accuracies": 0.7578125, | |
| "eval_rewards/chosen": -1.5020815134048462, | |
| "eval_rewards/margins": 0.8694581985473633, | |
| "eval_rewards/rejected": -2.371539831161499, | |
| "eval_runtime": 43.5081, | |
| "eval_samples_per_second": 45.968, | |
| "eval_steps_per_second": 0.735, | |
| "step": 300 | |
| }, | |
| { | |
| "debug/losses": 0.1321336030960083, | |
| "debug/policy_weights": 0.2647426724433899, | |
| "debug/raw_losses": 0.4956802725791931, | |
| "epoch": 0.65, | |
| "grad_norm": 7.197171827503445, | |
| "learning_rate": 1.6583128063291573e-07, | |
| "logits/chosen": -2.186096429824829, | |
| "logits/rejected": -2.185868501663208, | |
| "logps/chosen": -443.67803955078125, | |
| "logps/rejected": -486.25091552734375, | |
| "loss": 0.1268, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": -1.5781536102294922, | |
| "rewards/margins": 0.7728461027145386, | |
| "rewards/rejected": -2.3510000705718994, | |
| "step": 310 | |
| }, | |
| { | |
| "debug/losses": 0.13546636700630188, | |
| "debug/policy_weights": 0.2598869204521179, | |
| "debug/raw_losses": 0.5187257528305054, | |
| "epoch": 0.67, | |
| "grad_norm": 7.989288502873238, | |
| "learning_rate": 1.488723393865766e-07, | |
| "logits/chosen": -2.2410237789154053, | |
| "logits/rejected": -2.2062346935272217, | |
| "logps/chosen": -433.8523864746094, | |
| "logps/rejected": -467.3929138183594, | |
| "loss": 0.1308, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.493381142616272, | |
| "rewards/margins": 0.7878578901290894, | |
| "rewards/rejected": -2.2812390327453613, | |
| "step": 320 | |
| }, | |
| { | |
| "debug/losses": 0.14906349778175354, | |
| "debug/policy_weights": 0.2727554440498352, | |
| "debug/raw_losses": 0.5552006959915161, | |
| "epoch": 0.69, | |
| "grad_norm": 6.170487059058624, | |
| "learning_rate": 1.3245295796480788e-07, | |
| "logits/chosen": -2.236531972885132, | |
| "logits/rejected": -2.1864519119262695, | |
| "logps/chosen": -390.4073791503906, | |
| "logps/rejected": -459.62347412109375, | |
| "loss": 0.1411, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.373223066329956, | |
| "rewards/margins": 0.6836794018745422, | |
| "rewards/rejected": -2.0569024085998535, | |
| "step": 330 | |
| }, | |
| { | |
| "debug/losses": 0.1494503617286682, | |
| "debug/policy_weights": 0.28068724274635315, | |
| "debug/raw_losses": 0.4996885359287262, | |
| "epoch": 0.71, | |
| "grad_norm": 9.416700000227674, | |
| "learning_rate": 1.1666074087171627e-07, | |
| "logits/chosen": -2.2197887897491455, | |
| "logits/rejected": -2.1604628562927246, | |
| "logps/chosen": -418.9659118652344, | |
| "logps/rejected": -511.0225524902344, | |
| "loss": 0.1446, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.3679134845733643, | |
| "rewards/margins": 1.0408259630203247, | |
| "rewards/rejected": -2.4087395668029785, | |
| "step": 340 | |
| }, | |
| { | |
| "debug/losses": 0.12328938394784927, | |
| "debug/policy_weights": 0.24812527000904083, | |
| "debug/raw_losses": 0.48738375306129456, | |
| "epoch": 0.73, | |
| "grad_norm": 8.40666805347326, | |
| "learning_rate": 1.0157994641835734e-07, | |
| "logits/chosen": -2.172837018966675, | |
| "logits/rejected": -2.1325600147247314, | |
| "logps/chosen": -398.5860900878906, | |
| "logps/rejected": -471.38397216796875, | |
| "loss": 0.1286, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.4896047115325928, | |
| "rewards/margins": 0.9131780862808228, | |
| "rewards/rejected": -2.402782917022705, | |
| "step": 350 | |
| }, | |
| { | |
| "debug/losses": 0.1280422806739807, | |
| "debug/policy_weights": 0.2567124366760254, | |
| "debug/raw_losses": 0.5079829692840576, | |
| "epoch": 0.75, | |
| "grad_norm": 7.5652554505895955, | |
| "learning_rate": 8.729103716819111e-08, | |
| "logits/chosen": -2.230677366256714, | |
| "logits/rejected": -2.138909101486206, | |
| "logps/chosen": -448.53179931640625, | |
| "logps/rejected": -494.7422790527344, | |
| "loss": 0.1317, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -1.5306211709976196, | |
| "rewards/margins": 0.8633092641830444, | |
| "rewards/rejected": -2.393930435180664, | |
| "step": 360 | |
| }, | |
| { | |
| "debug/losses": 0.13079939782619476, | |
| "debug/policy_weights": 0.23427358269691467, | |
| "debug/raw_losses": 0.5890456438064575, | |
| "epoch": 0.77, | |
| "grad_norm": 9.87793971826204, | |
| "learning_rate": 7.387025063449081e-08, | |
| "logits/chosen": -2.167722225189209, | |
| "logits/rejected": -2.1336138248443604, | |
| "logps/chosen": -432.41314697265625, | |
| "logps/rejected": -470.364501953125, | |
| "loss": 0.1313, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.6386140584945679, | |
| "rewards/margins": 0.7472943067550659, | |
| "rewards/rejected": -2.385908365249634, | |
| "step": 370 | |
| }, | |
| { | |
| "debug/losses": 0.1057448610663414, | |
| "debug/policy_weights": 0.2194373905658722, | |
| "debug/raw_losses": 0.5027541518211365, | |
| "epoch": 0.79, | |
| "grad_norm": 7.002597773553437, | |
| "learning_rate": 6.138919252022435e-08, | |
| "logits/chosen": -2.093229055404663, | |
| "logits/rejected": -2.0845484733581543, | |
| "logps/chosen": -409.2686462402344, | |
| "logps/rejected": -525.614013671875, | |
| "loss": 0.1212, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.7782268524169922, | |
| "rewards/margins": 0.96544349193573, | |
| "rewards/rejected": -2.743670701980591, | |
| "step": 380 | |
| }, | |
| { | |
| "debug/losses": 0.14649717509746552, | |
| "debug/policy_weights": 0.27705007791519165, | |
| "debug/raw_losses": 0.5246635675430298, | |
| "epoch": 0.82, | |
| "grad_norm": 7.870379900484857, | |
| "learning_rate": 4.991445467064689e-08, | |
| "logits/chosen": -2.1749279499053955, | |
| "logits/rejected": -2.138232946395874, | |
| "logps/chosen": -439.6005859375, | |
| "logps/rejected": -510.23260498046875, | |
| "loss": 0.1384, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.4645540714263916, | |
| "rewards/margins": 0.8761545419692993, | |
| "rewards/rejected": -2.3407082557678223, | |
| "step": 390 | |
| }, | |
| { | |
| "debug/losses": 0.1481957882642746, | |
| "debug/policy_weights": 0.25738272070884705, | |
| "debug/raw_losses": 0.5208276510238647, | |
| "epoch": 0.84, | |
| "grad_norm": 7.317947546801369, | |
| "learning_rate": 3.9507259776993954e-08, | |
| "logits/chosen": -2.1648905277252197, | |
| "logits/rejected": -2.1050162315368652, | |
| "logps/chosen": -416.4757385253906, | |
| "logps/rejected": -496.00927734375, | |
| "loss": 0.1398, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.536012887954712, | |
| "rewards/margins": 0.8311750292778015, | |
| "rewards/rejected": -2.367187976837158, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "eval_debug/losses": 0.13974608480930328, | |
| "eval_debug/policy_weights": 0.28152135014533997, | |
| "eval_debug/raw_losses": 0.47273018956184387, | |
| "eval_logits/chosen": -2.2036194801330566, | |
| "eval_logits/rejected": -2.1694607734680176, | |
| "eval_logps/chosen": -386.73663330078125, | |
| "eval_logps/rejected": -483.9327392578125, | |
| "eval_loss": 0.14222779870033264, | |
| "eval_rewards/accuracies": 0.76171875, | |
| "eval_rewards/chosen": -1.3154058456420898, | |
| "eval_rewards/margins": 0.9613481163978577, | |
| "eval_rewards/rejected": -2.2767536640167236, | |
| "eval_runtime": 98.9464, | |
| "eval_samples_per_second": 20.213, | |
| "eval_steps_per_second": 0.323, | |
| "step": 400 | |
| }, | |
| { | |
| "debug/losses": 0.13129359483718872, | |
| "debug/policy_weights": 0.25963759422302246, | |
| "debug/raw_losses": 0.5042656660079956, | |
| "epoch": 0.86, | |
| "grad_norm": 7.093812921964164, | |
| "learning_rate": 3.022313472693447e-08, | |
| "logits/chosen": -2.2121942043304443, | |
| "logits/rejected": -2.1647019386291504, | |
| "logps/chosen": -432.4383850097656, | |
| "logps/rejected": -489.4990234375, | |
| "loss": 0.1404, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.4152858257293701, | |
| "rewards/margins": 0.8453054428100586, | |
| "rewards/rejected": -2.260591506958008, | |
| "step": 410 | |
| }, | |
| { | |
| "debug/losses": 0.15901803970336914, | |
| "debug/policy_weights": 0.2908507287502289, | |
| "debug/raw_losses": 0.5441660284996033, | |
| "epoch": 0.88, | |
| "grad_norm": 7.194889894590537, | |
| "learning_rate": 2.2111614344599684e-08, | |
| "logits/chosen": -2.1668639183044434, | |
| "logits/rejected": -2.1467199325561523, | |
| "logps/chosen": -427.835693359375, | |
| "logps/rejected": -492.3506774902344, | |
| "loss": 0.1412, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -1.4115890264511108, | |
| "rewards/margins": 0.840238094329834, | |
| "rewards/rejected": -2.2518272399902344, | |
| "step": 420 | |
| }, | |
| { | |
| "debug/losses": 0.1401083916425705, | |
| "debug/policy_weights": 0.2716709077358246, | |
| "debug/raw_losses": 0.49931907653808594, | |
| "epoch": 0.9, | |
| "grad_norm": 6.971935536096578, | |
| "learning_rate": 1.521597710086439e-08, | |
| "logits/chosen": -2.106959342956543, | |
| "logits/rejected": -2.051405668258667, | |
| "logps/chosen": -412.853759765625, | |
| "logps/rejected": -481.4043884277344, | |
| "loss": 0.1291, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -1.4160857200622559, | |
| "rewards/margins": 0.8900654911994934, | |
| "rewards/rejected": -2.3061509132385254, | |
| "step": 430 | |
| }, | |
| { | |
| "debug/losses": 0.14032089710235596, | |
| "debug/policy_weights": 0.28002962470054626, | |
| "debug/raw_losses": 0.4943667948246002, | |
| "epoch": 0.92, | |
| "grad_norm": 7.598363791526828, | |
| "learning_rate": 9.57301420397924e-09, | |
| "logits/chosen": -2.1854939460754395, | |
| "logits/rejected": -2.1297545433044434, | |
| "logps/chosen": -409.72625732421875, | |
| "logps/rejected": -484.60546875, | |
| "loss": 0.1401, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -1.338160753250122, | |
| "rewards/margins": 0.8834611773490906, | |
| "rewards/rejected": -2.2216219902038574, | |
| "step": 440 | |
| }, | |
| { | |
| "debug/losses": 0.1532571017742157, | |
| "debug/policy_weights": 0.28182727098464966, | |
| "debug/raw_losses": 0.5403295159339905, | |
| "epoch": 0.94, | |
| "grad_norm": 7.420969377583743, | |
| "learning_rate": 5.212833302556258e-09, | |
| "logits/chosen": -2.108179807662964, | |
| "logits/rejected": -2.0896925926208496, | |
| "logps/chosen": -421.1512756347656, | |
| "logps/rejected": -522.5540771484375, | |
| "loss": 0.1398, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.3998409509658813, | |
| "rewards/margins": 0.8198366165161133, | |
| "rewards/rejected": -2.219677448272705, | |
| "step": 450 | |
| }, | |
| { | |
| "debug/losses": 0.14014151692390442, | |
| "debug/policy_weights": 0.27090734243392944, | |
| "debug/raw_losses": 0.5257419347763062, | |
| "epoch": 0.96, | |
| "grad_norm": 6.950959627699968, | |
| "learning_rate": 2.158697848236607e-09, | |
| "logits/chosen": -2.1658027172088623, | |
| "logits/rejected": -2.1314563751220703, | |
| "logps/chosen": -402.38995361328125, | |
| "logps/rejected": -448.8475646972656, | |
| "loss": 0.1362, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.3720080852508545, | |
| "rewards/margins": 0.7829836010932922, | |
| "rewards/rejected": -2.154991626739502, | |
| "step": 460 | |
| }, | |
| { | |
| "debug/losses": 0.14437520503997803, | |
| "debug/policy_weights": 0.27037832140922546, | |
| "debug/raw_losses": 0.5343233942985535, | |
| "epoch": 0.98, | |
| "grad_norm": 8.44671954335882, | |
| "learning_rate": 4.269029751107489e-10, | |
| "logits/chosen": -2.151285171508789, | |
| "logits/rejected": -2.119642972946167, | |
| "logps/chosen": -403.640625, | |
| "logps/rejected": -483.1455078125, | |
| "loss": 0.1371, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.3858269453048706, | |
| "rewards/margins": 0.757634699344635, | |
| "rewards/rejected": -2.1434617042541504, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 478, | |
| "total_flos": 0.0, | |
| "train_loss": 0.17903520272865456, | |
| "train_runtime": 4631.245, | |
| "train_samples_per_second": 13.2, | |
| "train_steps_per_second": 0.103 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 478, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |