| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 3.0, | |
| "eval_steps": 100, | |
| "global_step": 1065, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "dpo_losses": 0.6931471824645996, | |
| "epoch": 0.0, | |
| "grad_norm": 2.1187342100202096, | |
| "learning_rate": 4.6728971962616815e-09, | |
| "logits/chosen": -2.8477635383605957, | |
| "logits/rejected": -2.8469698429107666, | |
| "logps/chosen": -522.6112670898438, | |
| "logps/rejected": -359.48583984375, | |
| "loss": 0.6931, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/margins_max": 0.0, | |
| "rewards/margins_min": 0.0, | |
| "rewards/margins_std": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "dpo_losses": 0.6926888227462769, | |
| "epoch": 0.03, | |
| "grad_norm": 24.895678335901746, | |
| "learning_rate": 4.672897196261682e-08, | |
| "logits/chosen": -2.9204907417297363, | |
| "logits/rejected": -2.7960145473480225, | |
| "logps/chosen": -313.4462890625, | |
| "logps/rejected": -170.4320068359375, | |
| "loss": 0.6989, | |
| "positive_losses": 0.036266010254621506, | |
| "rewards/accuracies": 0.3888888955116272, | |
| "rewards/chosen": 0.00024180197215173393, | |
| "rewards/margins": 0.0009183046640828252, | |
| "rewards/margins_max": 0.0020684306509792805, | |
| "rewards/margins_min": -0.0002318212646059692, | |
| "rewards/margins_std": 0.0016265236772596836, | |
| "rewards/rejected": -0.0006765025900676847, | |
| "step": 10 | |
| }, | |
| { | |
| "dpo_losses": 0.6935218572616577, | |
| "epoch": 0.06, | |
| "grad_norm": 26.261976434571714, | |
| "learning_rate": 9.345794392523364e-08, | |
| "logits/chosen": -2.764099597930908, | |
| "logits/rejected": -2.7112996578216553, | |
| "logps/chosen": -381.15252685546875, | |
| "logps/rejected": -244.30020141601562, | |
| "loss": 0.702, | |
| "positive_losses": 0.120574951171875, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -0.00042650121031329036, | |
| "rewards/margins": -0.0007463769870810211, | |
| "rewards/margins_max": 0.0008523863507434726, | |
| "rewards/margins_min": -0.0023451403249055147, | |
| "rewards/margins_std": 0.0022609930019825697, | |
| "rewards/rejected": 0.0003198757185600698, | |
| "step": 20 | |
| }, | |
| { | |
| "dpo_losses": 0.6929606795310974, | |
| "epoch": 0.08, | |
| "grad_norm": 26.568307039870714, | |
| "learning_rate": 1.4018691588785045e-07, | |
| "logits/chosen": -2.87646484375, | |
| "logits/rejected": -2.8245913982391357, | |
| "logps/chosen": -375.969970703125, | |
| "logps/rejected": -252.5666961669922, | |
| "loss": 0.6964, | |
| "positive_losses": 0.026204681023955345, | |
| "rewards/accuracies": 0.3499999940395355, | |
| "rewards/chosen": 0.0013370837550610304, | |
| "rewards/margins": 0.0003754205536097288, | |
| "rewards/margins_max": 0.001665195683017373, | |
| "rewards/margins_min": -0.00091435422655195, | |
| "rewards/margins_std": 0.0018240170320495963, | |
| "rewards/rejected": 0.0009616632014513016, | |
| "step": 30 | |
| }, | |
| { | |
| "dpo_losses": 0.6924411058425903, | |
| "epoch": 0.11, | |
| "grad_norm": 10.124655258526282, | |
| "learning_rate": 1.8691588785046729e-07, | |
| "logits/chosen": -2.7380053997039795, | |
| "logits/rejected": -2.7716264724731445, | |
| "logps/chosen": -306.62396240234375, | |
| "logps/rejected": -317.3713684082031, | |
| "loss": 0.6942, | |
| "positive_losses": 0.01987609826028347, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.003017005743458867, | |
| "rewards/margins": 0.0014187573688104749, | |
| "rewards/margins_max": 0.0038917693309485912, | |
| "rewards/margins_min": -0.0010542543604969978, | |
| "rewards/margins_std": 0.003497367026284337, | |
| "rewards/rejected": 0.001598248491063714, | |
| "step": 40 | |
| }, | |
| { | |
| "dpo_losses": 0.6920477151870728, | |
| "epoch": 0.14, | |
| "grad_norm": 2.3619576775282343, | |
| "learning_rate": 2.336448598130841e-07, | |
| "logits/chosen": -2.7991483211517334, | |
| "logits/rejected": -2.7094950675964355, | |
| "logps/chosen": -244.6391143798828, | |
| "logps/rejected": -173.8690643310547, | |
| "loss": 0.6924, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": 0.005839168094098568, | |
| "rewards/margins": 0.0022048726677894592, | |
| "rewards/margins_max": 0.004601255524903536, | |
| "rewards/margins_min": -0.00019151013111695647, | |
| "rewards/margins_std": 0.0033889967016875744, | |
| "rewards/rejected": 0.0036342956591397524, | |
| "step": 50 | |
| }, | |
| { | |
| "dpo_losses": 0.6909143328666687, | |
| "epoch": 0.17, | |
| "grad_norm": 2.3567546921207154, | |
| "learning_rate": 2.803738317757009e-07, | |
| "logits/chosen": -2.7648746967315674, | |
| "logits/rejected": -2.6973958015441895, | |
| "logps/chosen": -266.65625, | |
| "logps/rejected": -235.16543579101562, | |
| "loss": 0.6914, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.0077917128801345825, | |
| "rewards/margins": 0.004477448761463165, | |
| "rewards/margins_max": 0.006284839008003473, | |
| "rewards/margins_min": 0.0026700585149228573, | |
| "rewards/margins_std": 0.0025560357607901096, | |
| "rewards/rejected": 0.0033142641186714172, | |
| "step": 60 | |
| }, | |
| { | |
| "dpo_losses": 0.6901382207870483, | |
| "epoch": 0.2, | |
| "grad_norm": 1.8400876800751589, | |
| "learning_rate": 3.271028037383177e-07, | |
| "logits/chosen": -2.6974568367004395, | |
| "logits/rejected": -2.699871063232422, | |
| "logps/chosen": -332.7781677246094, | |
| "logps/rejected": -214.24301147460938, | |
| "loss": 0.6902, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.01162528432905674, | |
| "rewards/margins": 0.006035626865923405, | |
| "rewards/margins_max": 0.010061298497021198, | |
| "rewards/margins_min": 0.0020099543035030365, | |
| "rewards/margins_std": 0.005693159066140652, | |
| "rewards/rejected": 0.005589658860117197, | |
| "step": 70 | |
| }, | |
| { | |
| "dpo_losses": 0.688714861869812, | |
| "epoch": 0.23, | |
| "grad_norm": 4.087198387473911, | |
| "learning_rate": 3.7383177570093457e-07, | |
| "logits/chosen": -2.944587469100952, | |
| "logits/rejected": -2.859727382659912, | |
| "logps/chosen": -335.75775146484375, | |
| "logps/rejected": -286.72412109375, | |
| "loss": 0.6886, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.012916642241179943, | |
| "rewards/margins": 0.00890201423317194, | |
| "rewards/margins_max": 0.013774129562079906, | |
| "rewards/margins_min": 0.004029898438602686, | |
| "rewards/margins_std": 0.006890212185680866, | |
| "rewards/rejected": 0.004014627076685429, | |
| "step": 80 | |
| }, | |
| { | |
| "dpo_losses": 0.6876496076583862, | |
| "epoch": 0.25, | |
| "grad_norm": 2.4851076976994455, | |
| "learning_rate": 4.205607476635514e-07, | |
| "logits/chosen": -2.777012586593628, | |
| "logits/rejected": -2.767564296722412, | |
| "logps/chosen": -275.3215637207031, | |
| "logps/rejected": -202.66746520996094, | |
| "loss": 0.6869, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.01653674617409706, | |
| "rewards/margins": 0.011065873317420483, | |
| "rewards/margins_max": 0.019997073337435722, | |
| "rewards/margins_min": 0.002134673995897174, | |
| "rewards/margins_std": 0.012630623765289783, | |
| "rewards/rejected": 0.005470870994031429, | |
| "step": 90 | |
| }, | |
| { | |
| "dpo_losses": 0.6823551058769226, | |
| "epoch": 0.28, | |
| "grad_norm": 2.301867398682679, | |
| "learning_rate": 4.672897196261682e-07, | |
| "logits/chosen": -2.96421480178833, | |
| "logits/rejected": -2.8904025554656982, | |
| "logps/chosen": -455.4552307128906, | |
| "logps/rejected": -341.5201416015625, | |
| "loss": 0.6835, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.028032690286636353, | |
| "rewards/margins": 0.02175181731581688, | |
| "rewards/margins_max": 0.03047388233244419, | |
| "rewards/margins_min": 0.013029751367866993, | |
| "rewards/margins_std": 0.01233486458659172, | |
| "rewards/rejected": 0.00628087529912591, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "eval_dpo_losses": 0.6916564702987671, | |
| "eval_logits/chosen": -2.854139804840088, | |
| "eval_logits/rejected": -2.808910608291626, | |
| "eval_logps/chosen": -284.305908203125, | |
| "eval_logps/rejected": -258.5689392089844, | |
| "eval_loss": 0.6964598298072815, | |
| "eval_positive_losses": 0.0435669906437397, | |
| "eval_rewards/accuracies": 0.5833333134651184, | |
| "eval_rewards/chosen": 0.009152961894869804, | |
| "eval_rewards/margins": 0.0030159971211105585, | |
| "eval_rewards/margins_max": 0.015517139807343483, | |
| "eval_rewards/margins_min": -0.007612254936248064, | |
| "eval_rewards/margins_std": 0.01032496802508831, | |
| "eval_rewards/rejected": 0.006136965472251177, | |
| "eval_runtime": 380.9273, | |
| "eval_samples_per_second": 5.25, | |
| "eval_steps_per_second": 0.165, | |
| "step": 100 | |
| }, | |
| { | |
| "dpo_losses": 0.6818417310714722, | |
| "epoch": 0.31, | |
| "grad_norm": 2.49767805990243, | |
| "learning_rate": 4.999879018839287e-07, | |
| "logits/chosen": -2.851139545440674, | |
| "logits/rejected": -2.726069688796997, | |
| "logps/chosen": -324.7695007324219, | |
| "logps/rejected": -245.7756805419922, | |
| "loss": 0.6789, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.029296237975358963, | |
| "rewards/margins": 0.022873710840940475, | |
| "rewards/margins_max": 0.03740672022104263, | |
| "rewards/margins_min": 0.008340701460838318, | |
| "rewards/margins_std": 0.020552778616547585, | |
| "rewards/rejected": 0.006422528065741062, | |
| "step": 110 | |
| }, | |
| { | |
| "dpo_losses": 0.6763116717338562, | |
| "epoch": 0.34, | |
| "grad_norm": 2.0754455271947125, | |
| "learning_rate": 4.997728568369408e-07, | |
| "logits/chosen": -2.9743714332580566, | |
| "logits/rejected": -2.9403293132781982, | |
| "logps/chosen": -367.7530212402344, | |
| "logps/rejected": -330.84222412109375, | |
| "loss": 0.6754, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.041877757757902145, | |
| "rewards/margins": 0.03419335186481476, | |
| "rewards/margins_max": 0.04843282699584961, | |
| "rewards/margins_min": 0.019953874871134758, | |
| "rewards/margins_std": 0.02013765648007393, | |
| "rewards/rejected": 0.0076844110153615475, | |
| "step": 120 | |
| }, | |
| { | |
| "dpo_losses": 0.6634218692779541, | |
| "epoch": 0.37, | |
| "grad_norm": 2.0479204842061978, | |
| "learning_rate": 4.992892309373227e-07, | |
| "logits/chosen": -2.8403239250183105, | |
| "logits/rejected": -2.7551536560058594, | |
| "logps/chosen": -384.84814453125, | |
| "logps/rejected": -258.2791442871094, | |
| "loss": 0.6672, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.061241261661052704, | |
| "rewards/margins": 0.06080120801925659, | |
| "rewards/margins_max": 0.08230480551719666, | |
| "rewards/margins_min": 0.039297617971897125, | |
| "rewards/margins_std": 0.03041067160665989, | |
| "rewards/rejected": 0.00044005707604810596, | |
| "step": 130 | |
| }, | |
| { | |
| "dpo_losses": 0.6647445559501648, | |
| "epoch": 0.39, | |
| "grad_norm": 1.7367015733096829, | |
| "learning_rate": 4.985375442281968e-07, | |
| "logits/chosen": -2.8128199577331543, | |
| "logits/rejected": -2.783970355987549, | |
| "logps/chosen": -345.33953857421875, | |
| "logps/rejected": -231.666259765625, | |
| "loss": 0.666, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.05871708318591118, | |
| "rewards/margins": 0.058249205350875854, | |
| "rewards/margins_max": 0.0843496099114418, | |
| "rewards/margins_min": 0.03214879333972931, | |
| "rewards/margins_std": 0.03691155090928078, | |
| "rewards/rejected": 0.0004678791738115251, | |
| "step": 140 | |
| }, | |
| { | |
| "dpo_losses": 0.6681698560714722, | |
| "epoch": 0.42, | |
| "grad_norm": 2.1268042590236584, | |
| "learning_rate": 4.975186049985817e-07, | |
| "logits/chosen": -2.8787732124328613, | |
| "logits/rejected": -2.80892276763916, | |
| "logps/chosen": -291.04962158203125, | |
| "logps/rejected": -247.11825561523438, | |
| "loss": 0.6614, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.05772637203335762, | |
| "rewards/margins": 0.051212601363658905, | |
| "rewards/margins_max": 0.08002050220966339, | |
| "rewards/margins_min": 0.022404693067073822, | |
| "rewards/margins_std": 0.04074053093791008, | |
| "rewards/rejected": 0.006513768341392279, | |
| "step": 150 | |
| }, | |
| { | |
| "dpo_losses": 0.6561421751976013, | |
| "epoch": 0.45, | |
| "grad_norm": 1.7754914073644856, | |
| "learning_rate": 4.962335089142375e-07, | |
| "logits/chosen": -2.888826608657837, | |
| "logits/rejected": -2.7711613178253174, | |
| "logps/chosen": -323.6219787597656, | |
| "logps/rejected": -242.70925903320312, | |
| "loss": 0.6537, | |
| "positive_losses": 0.020649338141083717, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.08097714930772781, | |
| "rewards/margins": 0.07674388587474823, | |
| "rewards/margins_max": 0.12844815850257874, | |
| "rewards/margins_min": 0.02503962442278862, | |
| "rewards/margins_std": 0.07312087714672089, | |
| "rewards/rejected": 0.004233261104673147, | |
| "step": 160 | |
| }, | |
| { | |
| "dpo_losses": 0.6377928853034973, | |
| "epoch": 0.48, | |
| "grad_norm": 1.8314743659672637, | |
| "learning_rate": 4.946836378394966e-07, | |
| "logits/chosen": -2.927858591079712, | |
| "logits/rejected": -2.771623134613037, | |
| "logps/chosen": -388.8592224121094, | |
| "logps/rejected": -233.54714965820312, | |
| "loss": 0.6505, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.11622031778097153, | |
| "rewards/margins": 0.11623603105545044, | |
| "rewards/margins_max": 0.1615394800901413, | |
| "rewards/margins_min": 0.07093258202075958, | |
| "rewards/margins_std": 0.0640687644481659, | |
| "rewards/rejected": -1.5713460015831515e-05, | |
| "step": 170 | |
| }, | |
| { | |
| "dpo_losses": 0.6507928371429443, | |
| "epoch": 0.51, | |
| "grad_norm": 2.090628503360293, | |
| "learning_rate": 4.92870658351344e-07, | |
| "logits/chosen": -2.827677011489868, | |
| "logits/rejected": -2.7344062328338623, | |
| "logps/chosen": -314.31219482421875, | |
| "logps/rejected": -378.66693115234375, | |
| "loss": 0.6443, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.09823790937662125, | |
| "rewards/margins": 0.08783474564552307, | |
| "rewards/margins_max": 0.11842919886112213, | |
| "rewards/margins_min": 0.05724028870463371, | |
| "rewards/margins_std": 0.043267086148262024, | |
| "rewards/rejected": 0.010403157211840153, | |
| "step": 180 | |
| }, | |
| { | |
| "dpo_losses": 0.6270108222961426, | |
| "epoch": 0.54, | |
| "grad_norm": 2.5880477915128974, | |
| "learning_rate": 4.90796519947347e-07, | |
| "logits/chosen": -2.7683603763580322, | |
| "logits/rejected": -2.600294589996338, | |
| "logps/chosen": -441.5703125, | |
| "logps/rejected": -221.9513702392578, | |
| "loss": 0.6377, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.14541961252689362, | |
| "rewards/margins": 0.13966473937034607, | |
| "rewards/margins_max": 0.2020079791545868, | |
| "rewards/margins_min": 0.07732154428958893, | |
| "rewards/margins_std": 0.08816662430763245, | |
| "rewards/rejected": 0.005754842888563871, | |
| "step": 190 | |
| }, | |
| { | |
| "dpo_losses": 0.6278845071792603, | |
| "epoch": 0.56, | |
| "grad_norm": 6.409126045529177, | |
| "learning_rate": 4.88463452949359e-07, | |
| "logits/chosen": -2.9043941497802734, | |
| "logits/rejected": -2.8018863201141357, | |
| "logps/chosen": -317.3318786621094, | |
| "logps/rejected": -204.9638671875, | |
| "loss": 0.6367, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.13463780283927917, | |
| "rewards/margins": 0.13884270191192627, | |
| "rewards/margins_max": 0.20423416793346405, | |
| "rewards/margins_min": 0.07345118373632431, | |
| "rewards/margins_std": 0.09247754514217377, | |
| "rewards/rejected": -0.004204885568469763, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "eval_dpo_losses": 0.6862879395484924, | |
| "eval_logits/chosen": -2.8224480152130127, | |
| "eval_logits/rejected": -2.777904510498047, | |
| "eval_logps/chosen": -283.0695495605469, | |
| "eval_logps/rejected": -258.4836120605469, | |
| "eval_loss": 0.7632620930671692, | |
| "eval_positive_losses": 0.6990463733673096, | |
| "eval_rewards/accuracies": 0.5873016119003296, | |
| "eval_rewards/chosen": 0.02151678316295147, | |
| "eval_rewards/margins": 0.014526319690048695, | |
| "eval_rewards/margins_max": 0.07610397040843964, | |
| "eval_rewards/margins_min": -0.039124276489019394, | |
| "eval_rewards/margins_std": 0.051066700369119644, | |
| "eval_rewards/rejected": 0.006990462075918913, | |
| "eval_runtime": 353.0866, | |
| "eval_samples_per_second": 5.664, | |
| "eval_steps_per_second": 0.178, | |
| "step": 200 | |
| }, | |
| { | |
| "dpo_losses": 0.6355398893356323, | |
| "epoch": 0.59, | |
| "grad_norm": 2.0528134694636475, | |
| "learning_rate": 4.858739661052539e-07, | |
| "logits/chosen": -2.6923115253448486, | |
| "logits/rejected": -2.6051807403564453, | |
| "logps/chosen": -343.03765869140625, | |
| "logps/rejected": -267.228271484375, | |
| "loss": 0.6285, | |
| "positive_losses": 0.02857360802590847, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.1379883587360382, | |
| "rewards/margins": 0.12217812240123749, | |
| "rewards/margins_max": 0.2092159241437912, | |
| "rewards/margins_min": 0.035140346735715866, | |
| "rewards/margins_std": 0.12309001386165619, | |
| "rewards/rejected": 0.01581023633480072, | |
| "step": 210 | |
| }, | |
| { | |
| "dpo_losses": 0.6001917719841003, | |
| "epoch": 0.62, | |
| "grad_norm": 2.0452189261623355, | |
| "learning_rate": 4.830308438912687e-07, | |
| "logits/chosen": -2.9291024208068848, | |
| "logits/rejected": -2.7718849182128906, | |
| "logps/chosen": -389.2041931152344, | |
| "logps/rejected": -278.2353515625, | |
| "loss": 0.6208, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.20065200328826904, | |
| "rewards/margins": 0.2000540941953659, | |
| "rewards/margins_max": 0.27747657895088196, | |
| "rewards/margins_min": 0.12263162434101105, | |
| "rewards/margins_std": 0.10949190706014633, | |
| "rewards/rejected": 0.0005978975677862763, | |
| "step": 220 | |
| }, | |
| { | |
| "dpo_losses": 0.6307692527770996, | |
| "epoch": 0.65, | |
| "grad_norm": 2.2006814811498345, | |
| "learning_rate": 4.799371435178545e-07, | |
| "logits/chosen": -2.8787121772766113, | |
| "logits/rejected": -2.7966089248657227, | |
| "logps/chosen": -366.04168701171875, | |
| "logps/rejected": -335.1198425292969, | |
| "loss": 0.611, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.14716704189777374, | |
| "rewards/margins": 0.13250373303890228, | |
| "rewards/margins_max": 0.20927949249744415, | |
| "rewards/margins_min": 0.05572795867919922, | |
| "rewards/margins_std": 0.10857733339071274, | |
| "rewards/rejected": 0.01466330885887146, | |
| "step": 230 | |
| }, | |
| { | |
| "dpo_losses": 0.6340683102607727, | |
| "epoch": 0.68, | |
| "grad_norm": 1.8268647373784295, | |
| "learning_rate": 4.765961916422574e-07, | |
| "logits/chosen": -2.884403944015503, | |
| "logits/rejected": -2.725059986114502, | |
| "logps/chosen": -327.7989196777344, | |
| "logps/rejected": -292.8590087890625, | |
| "loss": 0.6248, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.13348861038684845, | |
| "rewards/margins": 0.12396250665187836, | |
| "rewards/margins_max": 0.17216315865516663, | |
| "rewards/margins_min": 0.0757618397474289, | |
| "rewards/margins_std": 0.06816603988409042, | |
| "rewards/rejected": 0.009526104666292667, | |
| "step": 240 | |
| }, | |
| { | |
| "dpo_losses": 0.6013151407241821, | |
| "epoch": 0.7, | |
| "grad_norm": 1.9729455716823008, | |
| "learning_rate": 4.730115807913626e-07, | |
| "logits/chosen": -2.900844097137451, | |
| "logits/rejected": -2.716480016708374, | |
| "logps/chosen": -394.9379577636719, | |
| "logps/rejected": -255.08218383789062, | |
| "loss": 0.6072, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.20289082825183868, | |
| "rewards/margins": 0.19841626286506653, | |
| "rewards/margins_max": 0.2661207318305969, | |
| "rewards/margins_min": 0.13071177899837494, | |
| "rewards/margins_std": 0.09574858844280243, | |
| "rewards/rejected": 0.004474560730159283, | |
| "step": 250 | |
| }, | |
| { | |
| "dpo_losses": 0.6106809973716736, | |
| "epoch": 0.73, | |
| "grad_norm": 5.819181150531001, | |
| "learning_rate": 4.691871654986485e-07, | |
| "logits/chosen": -2.8338541984558105, | |
| "logits/rejected": -2.799828052520752, | |
| "logps/chosen": -303.6441345214844, | |
| "logps/rejected": -224.24734497070312, | |
| "loss": 0.6119, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.18178164958953857, | |
| "rewards/margins": 0.18184307217597961, | |
| "rewards/margins_max": 0.31147363781929016, | |
| "rewards/margins_min": 0.05221250653266907, | |
| "rewards/margins_std": 0.18332532048225403, | |
| "rewards/rejected": -6.143822974991053e-05, | |
| "step": 260 | |
| }, | |
| { | |
| "dpo_losses": 0.5980932116508484, | |
| "epoch": 0.76, | |
| "grad_norm": 2.0922326810258784, | |
| "learning_rate": 4.6512705815940536e-07, | |
| "logits/chosen": -2.8425519466400146, | |
| "logits/rejected": -2.702369451522827, | |
| "logps/chosen": -409.5565490722656, | |
| "logps/rejected": -233.59228515625, | |
| "loss": 0.6087, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.19440071284770966, | |
| "rewards/margins": 0.20493817329406738, | |
| "rewards/margins_max": 0.2935718894004822, | |
| "rewards/margins_min": 0.11630449444055557, | |
| "rewards/margins_std": 0.12534697353839874, | |
| "rewards/rejected": -0.010537461377680302, | |
| "step": 270 | |
| }, | |
| { | |
| "dpo_losses": 0.628231406211853, | |
| "epoch": 0.79, | |
| "grad_norm": 3.7503530971865384, | |
| "learning_rate": 4.6083562460867544e-07, | |
| "logits/chosen": -2.710557460784912, | |
| "logits/rejected": -2.6718039512634277, | |
| "logps/chosen": -314.34271240234375, | |
| "logps/rejected": -264.9143981933594, | |
| "loss": 0.608, | |
| "positive_losses": 0.09001044929027557, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.1473403126001358, | |
| "rewards/margins": 0.13957121968269348, | |
| "rewards/margins_max": 0.22123010456562042, | |
| "rewards/margins_min": 0.05791233107447624, | |
| "rewards/margins_std": 0.1154831051826477, | |
| "rewards/rejected": 0.0077690863981842995, | |
| "step": 280 | |
| }, | |
| { | |
| "dpo_losses": 0.5966510772705078, | |
| "epoch": 0.82, | |
| "grad_norm": 2.147411420458553, | |
| "learning_rate": 4.563174794266683e-07, | |
| "logits/chosen": -2.9081060886383057, | |
| "logits/rejected": -2.7455201148986816, | |
| "logps/chosen": -339.8738708496094, | |
| "logps/rejected": -270.620849609375, | |
| "loss": 0.6098, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.21971745789051056, | |
| "rewards/margins": 0.21189472079277039, | |
| "rewards/margins_max": 0.33527523279190063, | |
| "rewards/margins_min": 0.08851419389247894, | |
| "rewards/margins_std": 0.17448639869689941, | |
| "rewards/rejected": 0.00782275851815939, | |
| "step": 290 | |
| }, | |
| { | |
| "dpo_losses": 0.6003170013427734, | |
| "epoch": 0.85, | |
| "grad_norm": 6.556261857915724, | |
| "learning_rate": 4.515774809767012e-07, | |
| "logits/chosen": -2.829227924346924, | |
| "logits/rejected": -2.7758102416992188, | |
| "logps/chosen": -317.3044128417969, | |
| "logps/rejected": -312.4596862792969, | |
| "loss": 0.5913, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.20438948273658752, | |
| "rewards/margins": 0.20183344185352325, | |
| "rewards/margins_max": 0.29987072944641113, | |
| "rewards/margins_min": 0.10379616171121597, | |
| "rewards/margins_std": 0.13864566385746002, | |
| "rewards/rejected": 0.0025560318026691675, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "eval_dpo_losses": 0.6810439825057983, | |
| "eval_logits/chosen": -2.7852697372436523, | |
| "eval_logits/rejected": -2.7412352561950684, | |
| "eval_logps/chosen": -283.9922180175781, | |
| "eval_logps/rejected": -260.6202087402344, | |
| "eval_loss": 0.919795036315918, | |
| "eval_positive_losses": 2.2040839195251465, | |
| "eval_rewards/accuracies": 0.5714285969734192, | |
| "eval_rewards/chosen": 0.012289770878851414, | |
| "eval_rewards/margins": 0.02666497975587845, | |
| "eval_rewards/margins_max": 0.13577593863010406, | |
| "eval_rewards/margins_min": -0.06833065301179886, | |
| "eval_rewards/margins_std": 0.08987120538949966, | |
| "eval_rewards/rejected": -0.01437520980834961, | |
| "eval_runtime": 376.3495, | |
| "eval_samples_per_second": 5.314, | |
| "eval_steps_per_second": 0.167, | |
| "step": 300 | |
| }, | |
| { | |
| "dpo_losses": 0.5683969259262085, | |
| "epoch": 0.87, | |
| "grad_norm": 1.838064317269046, | |
| "learning_rate": 4.4662072618099887e-07, | |
| "logits/chosen": -2.98117995262146, | |
| "logits/rejected": -2.7507948875427246, | |
| "logps/chosen": -399.4131164550781, | |
| "logps/rejected": -263.9547424316406, | |
| "loss": 0.593, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2483425885438919, | |
| "rewards/margins": 0.2780510485172272, | |
| "rewards/margins_max": 0.41263166069984436, | |
| "rewards/margins_min": 0.14347048103809357, | |
| "rewards/margins_std": 0.19032566249370575, | |
| "rewards/rejected": -0.029708484187722206, | |
| "step": 310 | |
| }, | |
| { | |
| "dpo_losses": 0.5936921834945679, | |
| "epoch": 0.9, | |
| "grad_norm": 1.773863081765084, | |
| "learning_rate": 4.414525450399712e-07, | |
| "logits/chosen": -2.838667631149292, | |
| "logits/rejected": -2.7466790676116943, | |
| "logps/chosen": -326.25115966796875, | |
| "logps/rejected": -239.88632202148438, | |
| "loss": 0.5855, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2000167816877365, | |
| "rewards/margins": 0.21792948246002197, | |
| "rewards/margins_max": 0.3168962597846985, | |
| "rewards/margins_min": 0.11896270513534546, | |
| "rewards/margins_std": 0.1399601548910141, | |
| "rewards/rejected": -0.01791267842054367, | |
| "step": 320 | |
| }, | |
| { | |
| "dpo_losses": 0.5716263651847839, | |
| "epoch": 0.93, | |
| "grad_norm": 1.998569334452282, | |
| "learning_rate": 4.360784949008615e-07, | |
| "logits/chosen": -2.938047409057617, | |
| "logits/rejected": -2.8133039474487305, | |
| "logps/chosen": -355.54949951171875, | |
| "logps/rejected": -251.0133056640625, | |
| "loss": 0.5905, | |
| "positive_losses": 0.12526169419288635, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.24137118458747864, | |
| "rewards/margins": 0.2771782875061035, | |
| "rewards/margins_max": 0.4031391143798828, | |
| "rewards/margins_min": 0.1512174755334854, | |
| "rewards/margins_std": 0.17813549935817719, | |
| "rewards/rejected": -0.03580709546804428, | |
| "step": 330 | |
| }, | |
| { | |
| "dpo_losses": 0.6020129919052124, | |
| "epoch": 0.96, | |
| "grad_norm": 1.9680439727569183, | |
| "learning_rate": 4.305043544819289e-07, | |
| "logits/chosen": -2.8385097980499268, | |
| "logits/rejected": -2.6945478916168213, | |
| "logps/chosen": -348.60601806640625, | |
| "logps/rejected": -204.32705688476562, | |
| "loss": 0.5827, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.19889523088932037, | |
| "rewards/margins": 0.19913819432258606, | |
| "rewards/margins_max": 0.2865811586380005, | |
| "rewards/margins_min": 0.11169523000717163, | |
| "rewards/margins_std": 0.123663030564785, | |
| "rewards/rejected": -0.00024295822368003428, | |
| "step": 340 | |
| }, | |
| { | |
| "dpo_losses": 0.5496091842651367, | |
| "epoch": 0.99, | |
| "grad_norm": 4.932583836163482, | |
| "learning_rate": 4.247361176585903e-07, | |
| "logits/chosen": -2.8020033836364746, | |
| "logits/rejected": -2.7020020484924316, | |
| "logps/chosen": -401.37969970703125, | |
| "logps/rejected": -303.43603515625, | |
| "loss": 0.5792, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.27023789286613464, | |
| "rewards/margins": 0.32071441411972046, | |
| "rewards/margins_max": 0.4361411929130554, | |
| "rewards/margins_min": 0.2052876502275467, | |
| "rewards/margins_std": 0.16323810815811157, | |
| "rewards/rejected": -0.05047653242945671, | |
| "step": 350 | |
| }, | |
| { | |
| "dpo_losses": 0.5841382741928101, | |
| "epoch": 1.01, | |
| "grad_norm": 1.907764958469603, | |
| "learning_rate": 4.187799870182038e-07, | |
| "logits/chosen": -2.7761244773864746, | |
| "logits/rejected": -2.671306610107422, | |
| "logps/chosen": -325.5156555175781, | |
| "logps/rejected": -205.7819061279297, | |
| "loss": 0.5713, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.20690234005451202, | |
| "rewards/margins": 0.2369709461927414, | |
| "rewards/margins_max": 0.33839744329452515, | |
| "rewards/margins_min": 0.13554444909095764, | |
| "rewards/margins_std": 0.14343872666358948, | |
| "rewards/rejected": -0.03006860613822937, | |
| "step": 360 | |
| }, | |
| { | |
| "dpo_losses": 0.571262776851654, | |
| "epoch": 1.04, | |
| "grad_norm": 7.1801497317882195, | |
| "learning_rate": 4.126423671904236e-07, | |
| "logits/chosen": -2.667179822921753, | |
| "logits/rejected": -2.659519672393799, | |
| "logps/chosen": -327.2443542480469, | |
| "logps/rejected": -267.2635192871094, | |
| "loss": 0.566, | |
| "positive_losses": 0.06003761291503906, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2441311627626419, | |
| "rewards/margins": 0.2789308428764343, | |
| "rewards/margins_max": 0.43240299820899963, | |
| "rewards/margins_min": 0.12545865774154663, | |
| "rewards/margins_std": 0.217042475938797, | |
| "rewards/rejected": -0.03479967638850212, | |
| "step": 370 | |
| }, | |
| { | |
| "dpo_losses": 0.5604450106620789, | |
| "epoch": 1.07, | |
| "grad_norm": 3.976395938884955, | |
| "learning_rate": 4.0632985796030007e-07, | |
| "logits/chosen": -2.775261640548706, | |
| "logits/rejected": -2.5520777702331543, | |
| "logps/chosen": -365.10052490234375, | |
| "logps/rejected": -188.36569213867188, | |
| "loss": 0.5666, | |
| "positive_losses": 0.1160304993391037, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2588001787662506, | |
| "rewards/margins": 0.29767391085624695, | |
| "rewards/margins_max": 0.4550997316837311, | |
| "rewards/margins_min": 0.1402481645345688, | |
| "rewards/margins_std": 0.22263364493846893, | |
| "rewards/rejected": -0.03887376934289932, | |
| "step": 380 | |
| }, | |
| { | |
| "dpo_losses": 0.5191501379013062, | |
| "epoch": 1.1, | |
| "grad_norm": 2.0256079149805704, | |
| "learning_rate": 3.9984924717152713e-07, | |
| "logits/chosen": -2.7934913635253906, | |
| "logits/rejected": -2.7538201808929443, | |
| "logps/chosen": -368.53741455078125, | |
| "logps/rejected": -343.0877380371094, | |
| "loss": 0.568, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2897542417049408, | |
| "rewards/margins": 0.4042133390903473, | |
| "rewards/margins_max": 0.5862180590629578, | |
| "rewards/margins_min": 0.22220861911773682, | |
| "rewards/margins_std": 0.2573935389518738, | |
| "rewards/rejected": -0.11445906013250351, | |
| "step": 390 | |
| }, | |
| { | |
| "dpo_losses": 0.5777878761291504, | |
| "epoch": 1.13, | |
| "grad_norm": 1.913623993136942, | |
| "learning_rate": 3.932075034274723e-07, | |
| "logits/chosen": -2.720954418182373, | |
| "logits/rejected": -2.731393337249756, | |
| "logps/chosen": -278.46453857421875, | |
| "logps/rejected": -233.5457000732422, | |
| "loss": 0.5502, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2027483880519867, | |
| "rewards/margins": 0.26584392786026, | |
| "rewards/margins_max": 0.36782872676849365, | |
| "rewards/margins_min": 0.16385909914970398, | |
| "rewards/margins_std": 0.14422830939292908, | |
| "rewards/rejected": -0.06309551745653152, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 1.13, | |
| "eval_dpo_losses": 0.6769555807113647, | |
| "eval_logits/chosen": -2.7545130252838135, | |
| "eval_logits/rejected": -2.7112648487091064, | |
| "eval_logps/chosen": -285.1260681152344, | |
| "eval_logps/rejected": -262.7898864746094, | |
| "eval_loss": 1.0825976133346558, | |
| "eval_positive_losses": 3.7846007347106934, | |
| "eval_rewards/accuracies": 0.5753968358039856, | |
| "eval_rewards/chosen": 0.0009513738332316279, | |
| "eval_rewards/margins": 0.037023574113845825, | |
| "eval_rewards/margins_max": 0.18605393171310425, | |
| "eval_rewards/margins_min": -0.09629133343696594, | |
| "eval_rewards/margins_std": 0.12429077178239822, | |
| "eval_rewards/rejected": -0.036072202026844025, | |
| "eval_runtime": 389.163, | |
| "eval_samples_per_second": 5.139, | |
| "eval_steps_per_second": 0.162, | |
| "step": 400 | |
| }, | |
| { | |
| "dpo_losses": 0.5712305307388306, | |
| "epoch": 1.15, | |
| "grad_norm": 2.3137171866935726, | |
| "learning_rate": 3.8641176859783383e-07, | |
| "logits/chosen": -2.801839590072632, | |
| "logits/rejected": -2.7598752975463867, | |
| "logps/chosen": -269.1628723144531, | |
| "logps/rejected": -211.1710662841797, | |
| "loss": 0.5608, | |
| "positive_losses": 0.19508972764015198, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.2552756071090698, | |
| "rewards/margins": 0.2850131392478943, | |
| "rewards/margins_max": 0.5092954635620117, | |
| "rewards/margins_min": 0.06073073670268059, | |
| "rewards/margins_std": 0.31718316674232483, | |
| "rewards/rejected": -0.029737496748566628, | |
| "step": 410 | |
| }, | |
| { | |
| "dpo_losses": 0.5262824296951294, | |
| "epoch": 1.18, | |
| "grad_norm": 2.2394985330776582, | |
| "learning_rate": 3.7946935013898606e-07, | |
| "logits/chosen": -2.8580517768859863, | |
| "logits/rejected": -2.7411797046661377, | |
| "logps/chosen": -366.37841796875, | |
| "logps/rejected": -270.374755859375, | |
| "loss": 0.5707, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.33375903964042664, | |
| "rewards/margins": 0.3817841410636902, | |
| "rewards/margins_max": 0.5622913241386414, | |
| "rewards/margins_min": 0.20127694308757782, | |
| "rewards/margins_std": 0.255275696516037, | |
| "rewards/rejected": -0.048025064170360565, | |
| "step": 420 | |
| }, | |
| { | |
| "dpo_losses": 0.543947160243988, | |
| "epoch": 1.21, | |
| "grad_norm": 5.549999635713583, | |
| "learning_rate": 3.7238771323626817e-07, | |
| "logits/chosen": -2.782466173171997, | |
| "logits/rejected": -2.675513744354248, | |
| "logps/chosen": -357.595458984375, | |
| "logps/rejected": -268.3279724121094, | |
| "loss": 0.5395, | |
| "positive_losses": 0.051211167126894, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.2949376702308655, | |
| "rewards/margins": 0.3435710072517395, | |
| "rewards/margins_max": 0.5000001192092896, | |
| "rewards/margins_min": 0.18714189529418945, | |
| "rewards/margins_std": 0.22122418880462646, | |
| "rewards/rejected": -0.04863337427377701, | |
| "step": 430 | |
| }, | |
| { | |
| "dpo_losses": 0.5732239484786987, | |
| "epoch": 1.24, | |
| "grad_norm": 1.8704082850098978, | |
| "learning_rate": 3.651744727766676e-07, | |
| "logits/chosen": -2.7718758583068848, | |
| "logits/rejected": -2.657120704650879, | |
| "logps/chosen": -292.4672546386719, | |
| "logps/rejected": -199.0224609375, | |
| "loss": 0.5501, | |
| "positive_losses": 0.02040863037109375, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.22786328196525574, | |
| "rewards/margins": 0.2696700394153595, | |
| "rewards/margins_max": 0.39789730310440063, | |
| "rewards/margins_min": 0.14144271612167358, | |
| "rewards/margins_std": 0.1813407838344574, | |
| "rewards/rejected": -0.04180673882365227, | |
| "step": 440 | |
| }, | |
| { | |
| "dpo_losses": 0.5517903566360474, | |
| "epoch": 1.27, | |
| "grad_norm": 1.988829111938975, | |
| "learning_rate": 3.5783738516052897e-07, | |
| "logits/chosen": -2.73368501663208, | |
| "logits/rejected": -2.664750576019287, | |
| "logps/chosen": -314.8210144042969, | |
| "logps/rejected": -276.9835205078125, | |
| "loss": 0.555, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.23353490233421326, | |
| "rewards/margins": 0.32702386379241943, | |
| "rewards/margins_max": 0.47540155053138733, | |
| "rewards/margins_min": 0.17864616215229034, | |
| "rewards/margins_std": 0.20983779430389404, | |
| "rewards/rejected": -0.09348895400762558, | |
| "step": 450 | |
| }, | |
| { | |
| "dpo_losses": 0.5432690382003784, | |
| "epoch": 1.3, | |
| "grad_norm": 6.009256851534543, | |
| "learning_rate": 3.5038433996109404e-07, | |
| "logits/chosen": -2.723193645477295, | |
| "logits/rejected": -2.7120513916015625, | |
| "logps/chosen": -370.0165710449219, | |
| "logps/rejected": -410.7750549316406, | |
| "loss": 0.5516, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.25261393189430237, | |
| "rewards/margins": 0.33443617820739746, | |
| "rewards/margins_max": 0.4311138689517975, | |
| "rewards/margins_min": 0.23775847256183624, | |
| "rewards/margins_std": 0.1367228925228119, | |
| "rewards/rejected": -0.0818222239613533, | |
| "step": 460 | |
| }, | |
| { | |
| "dpo_losses": 0.5220767259597778, | |
| "epoch": 1.32, | |
| "grad_norm": 9.755904928259515, | |
| "learning_rate": 3.428233514408398e-07, | |
| "logits/chosen": -2.768638849258423, | |
| "logits/rejected": -2.6755402088165283, | |
| "logps/chosen": -328.5848388671875, | |
| "logps/rejected": -230.9825439453125, | |
| "loss": 0.538, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.28665998578071594, | |
| "rewards/margins": 0.38968348503112793, | |
| "rewards/margins_max": 0.48602980375289917, | |
| "rewards/margins_min": 0.2933371067047119, | |
| "rewards/margins_std": 0.13625434041023254, | |
| "rewards/rejected": -0.10302351415157318, | |
| "step": 470 | |
| }, | |
| { | |
| "dpo_losses": 0.4982558786869049, | |
| "epoch": 1.35, | |
| "grad_norm": 1.8535108656666373, | |
| "learning_rate": 3.3516254993373945e-07, | |
| "logits/chosen": -2.855764150619507, | |
| "logits/rejected": -2.696035861968994, | |
| "logps/chosen": -384.2411193847656, | |
| "logps/rejected": -291.2297668457031, | |
| "loss": 0.542, | |
| "positive_losses": 0.18444347381591797, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.3161051273345947, | |
| "rewards/margins": 0.46426716446876526, | |
| "rewards/margins_max": 0.6109346747398376, | |
| "rewards/margins_min": 0.3175995349884033, | |
| "rewards/margins_std": 0.2074192762374878, | |
| "rewards/rejected": -0.14816200733184814, | |
| "step": 480 | |
| }, | |
| { | |
| "dpo_losses": 0.5826688408851624, | |
| "epoch": 1.38, | |
| "grad_norm": 2.1891083390571136, | |
| "learning_rate": 3.274101731027105e-07, | |
| "logits/chosen": -2.7111635208129883, | |
| "logits/rejected": -2.5885300636291504, | |
| "logps/chosen": -197.87130737304688, | |
| "logps/rejected": -222.28286743164062, | |
| "loss": 0.5455, | |
| "positive_losses": 0.21304932236671448, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.19410201907157898, | |
| "rewards/margins": 0.2511723041534424, | |
| "rewards/margins_max": 0.4196406304836273, | |
| "rewards/margins_min": 0.08270399272441864, | |
| "rewards/margins_std": 0.23825016617774963, | |
| "rewards/rejected": -0.057070292532444, | |
| "step": 490 | |
| }, | |
| { | |
| "dpo_losses": 0.512485682964325, | |
| "epoch": 1.41, | |
| "grad_norm": 7.948297320675137, | |
| "learning_rate": 3.1957455708165314e-07, | |
| "logits/chosen": -2.642686367034912, | |
| "logits/rejected": -2.5784401893615723, | |
| "logps/chosen": -350.71331787109375, | |
| "logps/rejected": -246.94711303710938, | |
| "loss": 0.5398, | |
| "positive_losses": 0.1300731599330902, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.33491355180740356, | |
| "rewards/margins": 0.42528876662254333, | |
| "rewards/margins_max": 0.566541850566864, | |
| "rewards/margins_min": 0.28403571248054504, | |
| "rewards/margins_std": 0.1997620314359665, | |
| "rewards/rejected": -0.09037523716688156, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 1.41, | |
| "eval_dpo_losses": 0.6733708381652832, | |
| "eval_logits/chosen": -2.736778974533081, | |
| "eval_logits/rejected": -2.6934895515441895, | |
| "eval_logps/chosen": -284.954833984375, | |
| "eval_logps/rejected": -263.591796875, | |
| "eval_loss": 1.1571382284164429, | |
| "eval_positive_losses": 4.656679630279541, | |
| "eval_rewards/accuracies": 0.5833333134651184, | |
| "eval_rewards/chosen": 0.0026637099217623472, | |
| "eval_rewards/margins": 0.04675525426864624, | |
| "eval_rewards/margins_max": 0.23377186059951782, | |
| "eval_rewards/margins_min": -0.11662713438272476, | |
| "eval_rewards/margins_std": 0.15486779808998108, | |
| "eval_rewards/rejected": -0.04409153386950493, | |
| "eval_runtime": 348.5206, | |
| "eval_samples_per_second": 5.739, | |
| "eval_steps_per_second": 0.181, | |
| "step": 500 | |
| }, | |
| { | |
| "dpo_losses": 0.558113694190979, | |
| "epoch": 1.44, | |
| "grad_norm": 1.9204678229765098, | |
| "learning_rate": 3.116641275116018e-07, | |
| "logits/chosen": -2.4792568683624268, | |
| "logits/rejected": -2.477318286895752, | |
| "logps/chosen": -267.8980407714844, | |
| "logps/rejected": -321.2754821777344, | |
| "loss": 0.5399, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.24782295525074005, | |
| "rewards/margins": 0.30803701281547546, | |
| "rewards/margins_max": 0.4859069287776947, | |
| "rewards/margins_min": 0.13016708195209503, | |
| "rewards/margins_std": 0.25154608488082886, | |
| "rewards/rejected": -0.06021404266357422, | |
| "step": 510 | |
| }, | |
| { | |
| "dpo_losses": 0.5207396149635315, | |
| "epoch": 1.46, | |
| "grad_norm": 1.8920245074404747, | |
| "learning_rate": 3.036873904806295e-07, | |
| "logits/chosen": -2.7549643516540527, | |
| "logits/rejected": -2.6787781715393066, | |
| "logps/chosen": -313.6898498535156, | |
| "logps/rejected": -246.35202026367188, | |
| "loss": 0.5479, | |
| "positive_losses": 0.4201123118400574, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2792617380619049, | |
| "rewards/margins": 0.4105965197086334, | |
| "rewards/margins_max": 0.6103520393371582, | |
| "rewards/margins_min": 0.21084094047546387, | |
| "rewards/margins_std": 0.28249698877334595, | |
| "rewards/rejected": -0.13133473694324493, | |
| "step": 520 | |
| }, | |
| { | |
| "dpo_losses": 0.5030455589294434, | |
| "epoch": 1.49, | |
| "grad_norm": 2.1161532532804963, | |
| "learning_rate": 2.956529233772492e-07, | |
| "logits/chosen": -2.721666097640991, | |
| "logits/rejected": -2.7209858894348145, | |
| "logps/chosen": -350.1697998046875, | |
| "logps/rejected": -284.701416015625, | |
| "loss": 0.5383, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2883579134941101, | |
| "rewards/margins": 0.4384874701499939, | |
| "rewards/margins_max": 0.5584603548049927, | |
| "rewards/margins_min": 0.31851455569267273, | |
| "rewards/margins_std": 0.16966724395751953, | |
| "rewards/rejected": -0.15012958645820618, | |
| "step": 530 | |
| }, | |
| { | |
| "dpo_losses": 0.4899216592311859, | |
| "epoch": 1.52, | |
| "grad_norm": 8.076099370517953, | |
| "learning_rate": 2.875693656671431e-07, | |
| "logits/chosen": -2.8387975692749023, | |
| "logits/rejected": -2.7006285190582275, | |
| "logps/chosen": -364.0811462402344, | |
| "logps/rejected": -250.73196411132812, | |
| "loss": 0.5309, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3547210097312927, | |
| "rewards/margins": 0.4873575270175934, | |
| "rewards/margins_max": 0.6359044313430786, | |
| "rewards/margins_min": 0.3388107419013977, | |
| "rewards/margins_std": 0.2100769281387329, | |
| "rewards/rejected": -0.13263657689094543, | |
| "step": 540 | |
| }, | |
| { | |
| "dpo_losses": 0.5343499183654785, | |
| "epoch": 1.55, | |
| "grad_norm": 2.5715523833788465, | |
| "learning_rate": 2.794454096031429e-07, | |
| "logits/chosen": -2.7994258403778076, | |
| "logits/rejected": -2.7424252033233643, | |
| "logps/chosen": -283.88751220703125, | |
| "logps/rejected": -277.2805480957031, | |
| "loss": 0.5163, | |
| "positive_losses": 0.1207679733633995, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2706112265586853, | |
| "rewards/margins": 0.3728974759578705, | |
| "rewards/margins_max": 0.5763125419616699, | |
| "rewards/margins_min": 0.16948243975639343, | |
| "rewards/margins_std": 0.2876723110675812, | |
| "rewards/rejected": -0.10228625684976578, | |
| "step": 550 | |
| }, | |
| { | |
| "dpo_losses": 0.5260319709777832, | |
| "epoch": 1.58, | |
| "grad_norm": 2.100863996516755, | |
| "learning_rate": 2.7128979087844593e-07, | |
| "logits/chosen": -2.7390799522399902, | |
| "logits/rejected": -2.6754238605499268, | |
| "logps/chosen": -291.78973388671875, | |
| "logps/rejected": -352.7436218261719, | |
| "loss": 0.5334, | |
| "positive_losses": 0.1672992706298828, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.24893280863761902, | |
| "rewards/margins": 0.388366162776947, | |
| "rewards/margins_max": 0.5333099961280823, | |
| "rewards/margins_min": 0.24342235922813416, | |
| "rewards/margins_std": 0.2049814909696579, | |
| "rewards/rejected": -0.139433354139328, | |
| "step": 560 | |
| }, | |
| { | |
| "dpo_losses": 0.49240389466285706, | |
| "epoch": 1.61, | |
| "grad_norm": 8.972280412241831, | |
| "learning_rate": 2.6311127923312153e-07, | |
| "logits/chosen": -2.757660388946533, | |
| "logits/rejected": -2.627717971801758, | |
| "logps/chosen": -397.9381103515625, | |
| "logps/rejected": -329.2608947753906, | |
| "loss": 0.5165, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3426300883293152, | |
| "rewards/margins": 0.47666874527931213, | |
| "rewards/margins_max": 0.6256591081619263, | |
| "rewards/margins_min": 0.32767823338508606, | |
| "rewards/margins_std": 0.21070432662963867, | |
| "rewards/rejected": -0.13403865694999695, | |
| "step": 570 | |
| }, | |
| { | |
| "dpo_losses": 0.5157249569892883, | |
| "epoch": 1.63, | |
| "grad_norm": 1.8838498119092633, | |
| "learning_rate": 2.5491866902400565e-07, | |
| "logits/chosen": -2.7878096103668213, | |
| "logits/rejected": -2.70076060295105, | |
| "logps/chosen": -270.0621337890625, | |
| "logps/rejected": -235.17105102539062, | |
| "loss": 0.5244, | |
| "positive_losses": 0.14246292412281036, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.28403371572494507, | |
| "rewards/margins": 0.4212573170661926, | |
| "rewards/margins_max": 0.6702337861061096, | |
| "rewards/margins_min": 0.17228081822395325, | |
| "rewards/margins_std": 0.35210588574409485, | |
| "rewards/rejected": -0.13722361624240875, | |
| "step": 580 | |
| }, | |
| { | |
| "dpo_losses": 0.524344801902771, | |
| "epoch": 1.66, | |
| "grad_norm": 7.342743240499319, | |
| "learning_rate": 2.4672076976812543e-07, | |
| "logits/chosen": -2.6465201377868652, | |
| "logits/rejected": -2.559436082839966, | |
| "logps/chosen": -339.4754943847656, | |
| "logps/rejected": -302.15557861328125, | |
| "loss": 0.5432, | |
| "positive_losses": 0.1482059508562088, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2978511452674866, | |
| "rewards/margins": 0.3972291350364685, | |
| "rewards/margins_max": 0.5903550982475281, | |
| "rewards/margins_min": 0.2041032314300537, | |
| "rewards/margins_std": 0.27312135696411133, | |
| "rewards/rejected": -0.09937803447246552, | |
| "step": 590 | |
| }, | |
| { | |
| "dpo_losses": 0.5494340658187866, | |
| "epoch": 1.69, | |
| "grad_norm": 8.369464468282295, | |
| "learning_rate": 2.385263966698222e-07, | |
| "logits/chosen": -2.7804884910583496, | |
| "logits/rejected": -2.7123289108276367, | |
| "logps/chosen": -253.48031616210938, | |
| "logps/rejected": -268.9811096191406, | |
| "loss": 0.5293, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2517229914665222, | |
| "rewards/margins": 0.3363966941833496, | |
| "rewards/margins_max": 0.48714661598205566, | |
| "rewards/margins_min": 0.18564683198928833, | |
| "rewards/margins_std": 0.21319253742694855, | |
| "rewards/rejected": -0.08467370271682739, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 1.69, | |
| "eval_dpo_losses": 0.6702548265457153, | |
| "eval_logits/chosen": -2.7200992107391357, | |
| "eval_logits/rejected": -2.6766622066497803, | |
| "eval_logps/chosen": -285.0616149902344, | |
| "eval_logps/rejected": -264.5409851074219, | |
| "eval_loss": 1.2245166301727295, | |
| "eval_positive_losses": 5.373989582061768, | |
| "eval_rewards/accuracies": 0.591269850730896, | |
| "eval_rewards/chosen": 0.0015958804870024323, | |
| "eval_rewards/margins": 0.05517909303307533, | |
| "eval_rewards/margins_max": 0.2654685378074646, | |
| "eval_rewards/margins_min": -0.128387451171875, | |
| "eval_rewards/margins_std": 0.1752447932958603, | |
| "eval_rewards/rejected": -0.053583212196826935, | |
| "eval_runtime": 377.5968, | |
| "eval_samples_per_second": 5.297, | |
| "eval_steps_per_second": 0.167, | |
| "step": 600 | |
| }, | |
| { | |
| "dpo_losses": 0.5357104539871216, | |
| "epoch": 1.72, | |
| "grad_norm": 8.256557748215014, | |
| "learning_rate": 2.3034436114175838e-07, | |
| "logits/chosen": -2.6488845348358154, | |
| "logits/rejected": -2.538515567779541, | |
| "logps/chosen": -336.7628479003906, | |
| "logps/rejected": -277.02264404296875, | |
| "loss": 0.5238, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.28726598620414734, | |
| "rewards/margins": 0.37427616119384766, | |
| "rewards/margins_max": 0.564933717250824, | |
| "rewards/margins_min": 0.18361851572990417, | |
| "rewards/margins_std": 0.2696306109428406, | |
| "rewards/rejected": -0.08701014518737793, | |
| "step": 610 | |
| }, | |
| { | |
| "dpo_losses": 0.5276176929473877, | |
| "epoch": 1.75, | |
| "grad_norm": 2.4842449610043764, | |
| "learning_rate": 2.2218346133000264e-07, | |
| "logits/chosen": -2.6663379669189453, | |
| "logits/rejected": -2.5575757026672363, | |
| "logps/chosen": -281.8042297363281, | |
| "logps/rejected": -206.41580200195312, | |
| "loss": 0.5162, | |
| "positive_losses": 0.15401744842529297, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2947912812232971, | |
| "rewards/margins": 0.38691216707229614, | |
| "rewards/margins_max": 0.5189955234527588, | |
| "rewards/margins_min": 0.2548287510871887, | |
| "rewards/margins_std": 0.18679411709308624, | |
| "rewards/rejected": -0.09212087094783783, | |
| "step": 620 | |
| }, | |
| { | |
| "dpo_losses": 0.5108110308647156, | |
| "epoch": 1.77, | |
| "grad_norm": 9.294890068434233, | |
| "learning_rate": 2.1405247265337917e-07, | |
| "logits/chosen": -2.6686511039733887, | |
| "logits/rejected": -2.5705299377441406, | |
| "logps/chosen": -360.513671875, | |
| "logps/rejected": -233.205322265625, | |
| "loss": 0.5195, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3424997925758362, | |
| "rewards/margins": 0.4331623613834381, | |
| "rewards/margins_max": 0.5366414785385132, | |
| "rewards/margins_min": 0.3296832740306854, | |
| "rewards/margins_std": 0.14634151756763458, | |
| "rewards/rejected": -0.09066257625818253, | |
| "step": 630 | |
| }, | |
| { | |
| "dpo_losses": 0.5333245396614075, | |
| "epoch": 1.8, | |
| "grad_norm": 1.9615557786341105, | |
| "learning_rate": 2.0596013836725657e-07, | |
| "logits/chosen": -2.793367862701416, | |
| "logits/rejected": -2.739849090576172, | |
| "logps/chosen": -275.52984619140625, | |
| "logps/rejected": -210.9861297607422, | |
| "loss": 0.542, | |
| "positive_losses": 0.4148605465888977, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.26945605874061584, | |
| "rewards/margins": 0.38035932183265686, | |
| "rewards/margins_max": 0.5337552428245544, | |
| "rewards/margins_min": 0.2269633710384369, | |
| "rewards/margins_std": 0.21693463623523712, | |
| "rewards/rejected": -0.11090326309204102, | |
| "step": 640 | |
| }, | |
| { | |
| "dpo_losses": 0.5089551210403442, | |
| "epoch": 1.83, | |
| "grad_norm": 11.059235344334038, | |
| "learning_rate": 1.9791516016192213e-07, | |
| "logits/chosen": -2.844893217086792, | |
| "logits/rejected": -2.7059712409973145, | |
| "logps/chosen": -298.34588623046875, | |
| "logps/rejected": -231.402099609375, | |
| "loss": 0.5102, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.3605387806892395, | |
| "rewards/margins": 0.446432501077652, | |
| "rewards/margins_max": 0.6981122493743896, | |
| "rewards/margins_min": 0.1947527527809143, | |
| "rewards/margins_std": 0.35592886805534363, | |
| "rewards/rejected": -0.0858936682343483, | |
| "step": 650 | |
| }, | |
| { | |
| "dpo_losses": 0.5299183130264282, | |
| "epoch": 1.86, | |
| "grad_norm": 16.447544756456054, | |
| "learning_rate": 1.8992618880565036e-07, | |
| "logits/chosen": -2.5619029998779297, | |
| "logits/rejected": -2.5244908332824707, | |
| "logps/chosen": -294.64801025390625, | |
| "logps/rejected": -199.9069061279297, | |
| "loss": 0.54, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.293754518032074, | |
| "rewards/margins": 0.39233601093292236, | |
| "rewards/margins_max": 0.5898939371109009, | |
| "rewards/margins_min": 0.19477804005146027, | |
| "rewards/margins_std": 0.2793891131877899, | |
| "rewards/rejected": -0.0985814779996872, | |
| "step": 660 | |
| }, | |
| { | |
| "dpo_losses": 0.4770924150943756, | |
| "epoch": 1.89, | |
| "grad_norm": 14.950375677893724, | |
| "learning_rate": 1.8200181484252885e-07, | |
| "logits/chosen": -2.7151541709899902, | |
| "logits/rejected": -2.7282559871673584, | |
| "logps/chosen": -347.63665771484375, | |
| "logps/rejected": -335.4634094238281, | |
| "loss": 0.5076, | |
| "positive_losses": 0.011021423153579235, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3512026071548462, | |
| "rewards/margins": 0.532378077507019, | |
| "rewards/margins_max": 0.7549656629562378, | |
| "rewards/margins_min": 0.3097904622554779, | |
| "rewards/margins_std": 0.314786434173584, | |
| "rewards/rejected": -0.18117551505565643, | |
| "step": 670 | |
| }, | |
| { | |
| "dpo_losses": 0.48109301924705505, | |
| "epoch": 1.92, | |
| "grad_norm": 1.8077778304327392, | |
| "learning_rate": 1.7415055935504233e-07, | |
| "logits/chosen": -2.796618938446045, | |
| "logits/rejected": -2.6364896297454834, | |
| "logps/chosen": -357.8368835449219, | |
| "logps/rejected": -312.83050537109375, | |
| "loss": 0.5045, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3552249073982239, | |
| "rewards/margins": 0.5131222009658813, | |
| "rewards/margins_max": 0.6571098566055298, | |
| "rewards/margins_min": 0.3691345155239105, | |
| "rewards/margins_std": 0.2036292850971222, | |
| "rewards/rejected": -0.15789727866649628, | |
| "step": 680 | |
| }, | |
| { | |
| "dpo_losses": 0.5480272769927979, | |
| "epoch": 1.94, | |
| "grad_norm": 7.710262187848809, | |
| "learning_rate": 1.6638086480134952e-07, | |
| "logits/chosen": -2.6526730060577393, | |
| "logits/rejected": -2.5982117652893066, | |
| "logps/chosen": -211.858154296875, | |
| "logps/rejected": -139.07669067382812, | |
| "loss": 0.519, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.25107190012931824, | |
| "rewards/margins": 0.34679529070854187, | |
| "rewards/margins_max": 0.5567124485969543, | |
| "rewards/margins_min": 0.1368781328201294, | |
| "rewards/margins_std": 0.2968676686286926, | |
| "rewards/rejected": -0.09572339057922363, | |
| "step": 690 | |
| }, | |
| { | |
| "dpo_losses": 0.4729584753513336, | |
| "epoch": 1.97, | |
| "grad_norm": 13.004502323752208, | |
| "learning_rate": 1.5870108593710471e-07, | |
| "logits/chosen": -2.577094793319702, | |
| "logits/rejected": -2.474807024002075, | |
| "logps/chosen": -388.23529052734375, | |
| "logps/rejected": -229.945556640625, | |
| "loss": 0.5238, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.3872971832752228, | |
| "rewards/margins": 0.5365989804267883, | |
| "rewards/margins_max": 0.6989376544952393, | |
| "rewards/margins_min": 0.3742601275444031, | |
| "rewards/margins_std": 0.22958168387413025, | |
| "rewards/rejected": -0.14930173754692078, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 1.97, | |
| "eval_dpo_losses": 0.6682811975479126, | |
| "eval_logits/chosen": -2.7153635025024414, | |
| "eval_logits/rejected": -2.6725597381591797, | |
| "eval_logps/chosen": -287.1236877441406, | |
| "eval_logps/rejected": -267.1868896484375, | |
| "eval_loss": 1.3783056735992432, | |
| "eval_positive_losses": 6.938729763031006, | |
| "eval_rewards/accuracies": 0.60317462682724, | |
| "eval_rewards/chosen": -0.019024791195988655, | |
| "eval_rewards/margins": 0.061017535626888275, | |
| "eval_rewards/margins_max": 0.28910568356513977, | |
| "eval_rewards/margins_min": -0.14251713454723358, | |
| "eval_rewards/margins_std": 0.1922098994255066, | |
| "eval_rewards/rejected": -0.08004232496023178, | |
| "eval_runtime": 374.9449, | |
| "eval_samples_per_second": 5.334, | |
| "eval_steps_per_second": 0.168, | |
| "step": 700 | |
| }, | |
| { | |
| "dpo_losses": 0.5287314057350159, | |
| "epoch": 2.0, | |
| "grad_norm": 2.5225442066988153, | |
| "learning_rate": 1.5111948083158528e-07, | |
| "logits/chosen": -2.645501136779785, | |
| "logits/rejected": -2.5416979789733887, | |
| "logps/chosen": -277.56146240234375, | |
| "logps/rejected": -194.89093017578125, | |
| "loss": 0.509, | |
| "positive_losses": 0.014240646734833717, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2871246635913849, | |
| "rewards/margins": 0.39854830503463745, | |
| "rewards/margins_max": 0.6037707328796387, | |
| "rewards/margins_min": 0.19332581758499146, | |
| "rewards/margins_std": 0.2902284264564514, | |
| "rewards/rejected": -0.11142361164093018, | |
| "step": 710 | |
| }, | |
| { | |
| "dpo_losses": 0.4969090521335602, | |
| "epoch": 2.03, | |
| "grad_norm": 6.7685343277404435, | |
| "learning_rate": 1.4364420198778658e-07, | |
| "logits/chosen": -2.711702823638916, | |
| "logits/rejected": -2.667511224746704, | |
| "logps/chosen": -318.25323486328125, | |
| "logps/rejected": -349.77325439453125, | |
| "loss": 0.4995, | |
| "positive_losses": 0.2103443145751953, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.29361721873283386, | |
| "rewards/margins": 0.48742538690567017, | |
| "rewards/margins_max": 0.7825571298599243, | |
| "rewards/margins_min": 0.19229364395141602, | |
| "rewards/margins_std": 0.417379230260849, | |
| "rewards/rejected": -0.1938081681728363, | |
| "step": 720 | |
| }, | |
| { | |
| "dpo_losses": 0.46872028708457947, | |
| "epoch": 2.06, | |
| "grad_norm": 1.9152175221021666, | |
| "learning_rate": 1.3628328757603242e-07, | |
| "logits/chosen": -2.715259552001953, | |
| "logits/rejected": -2.639958143234253, | |
| "logps/chosen": -387.1348571777344, | |
| "logps/rejected": -270.98504638671875, | |
| "loss": 0.5078, | |
| "positive_losses": 0.07048721611499786, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.3561549186706543, | |
| "rewards/margins": 0.5383674502372742, | |
| "rewards/margins_max": 0.6862602829933167, | |
| "rewards/margins_min": 0.39047467708587646, | |
| "rewards/margins_std": 0.20915205776691437, | |
| "rewards/rejected": -0.18221257627010345, | |
| "step": 730 | |
| }, | |
| { | |
| "dpo_losses": 0.48394322395324707, | |
| "epoch": 2.08, | |
| "grad_norm": 1.9934270854366338, | |
| "learning_rate": 1.2904465279052723e-07, | |
| "logits/chosen": -2.743479013442993, | |
| "logits/rejected": -2.6603636741638184, | |
| "logps/chosen": -314.8957824707031, | |
| "logps/rejected": -239.50637817382812, | |
| "loss": 0.5064, | |
| "positive_losses": 0.24646854400634766, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.37057873606681824, | |
| "rewards/margins": 0.5130593776702881, | |
| "rewards/margins_max": 0.7771711945533752, | |
| "rewards/margins_min": 0.2489476501941681, | |
| "rewards/margins_std": 0.373510479927063, | |
| "rewards/rejected": -0.14248065650463104, | |
| "step": 740 | |
| }, | |
| { | |
| "dpo_losses": 0.5582190752029419, | |
| "epoch": 2.11, | |
| "grad_norm": 9.439233885890266, | |
| "learning_rate": 1.219360813381446e-07, | |
| "logits/chosen": -2.647359848022461, | |
| "logits/rejected": -2.597470760345459, | |
| "logps/chosen": -167.36611938476562, | |
| "logps/rejected": -147.13504028320312, | |
| "loss": 0.4945, | |
| "positive_losses": 0.07765503227710724, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.21209602057933807, | |
| "rewards/margins": 0.31703463196754456, | |
| "rewards/margins_max": 0.4738802909851074, | |
| "rewards/margins_min": 0.1601889729499817, | |
| "rewards/margins_std": 0.22181324660778046, | |
| "rewards/rejected": -0.10493861138820648, | |
| "step": 750 | |
| }, | |
| { | |
| "dpo_losses": 0.4905478060245514, | |
| "epoch": 2.14, | |
| "grad_norm": 11.006048100367254, | |
| "learning_rate": 1.149652170686039e-07, | |
| "logits/chosen": -2.7402448654174805, | |
| "logits/rejected": -2.6278908252716064, | |
| "logps/chosen": -300.5604553222656, | |
| "logps/rejected": -279.05877685546875, | |
| "loss": 0.5065, | |
| "positive_losses": 0.7673536539077759, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.3250073492527008, | |
| "rewards/margins": 0.48759451508522034, | |
| "rewards/margins_max": 0.6997831463813782, | |
| "rewards/margins_min": 0.2754059433937073, | |
| "rewards/margins_std": 0.30007994174957275, | |
| "rewards/rejected": -0.1625872105360031, | |
| "step": 760 | |
| }, | |
| { | |
| "dpo_losses": 0.4878745675086975, | |
| "epoch": 2.17, | |
| "grad_norm": 2.041360663493708, | |
| "learning_rate": 1.0813955575503587e-07, | |
| "logits/chosen": -2.6781890392303467, | |
| "logits/rejected": -2.673537254333496, | |
| "logps/chosen": -303.26922607421875, | |
| "logps/rejected": -278.1116638183594, | |
| "loss": 0.4955, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.303671658039093, | |
| "rewards/margins": 0.5081648230552673, | |
| "rewards/margins_max": 0.7443748712539673, | |
| "rewards/margins_min": 0.27195480465888977, | |
| "rewards/margins_std": 0.3340514898300171, | |
| "rewards/rejected": -0.2044931948184967, | |
| "step": 770 | |
| }, | |
| { | |
| "dpo_losses": 0.5306052565574646, | |
| "epoch": 2.2, | |
| "grad_norm": 12.846225026696498, | |
| "learning_rate": 1.0146643703377486e-07, | |
| "logits/chosen": -2.753620147705078, | |
| "logits/rejected": -2.623445749282837, | |
| "logps/chosen": -295.22698974609375, | |
| "logps/rejected": -235.02810668945312, | |
| "loss": 0.5182, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.31814926862716675, | |
| "rewards/margins": 0.3883030414581299, | |
| "rewards/margins_max": 0.6206892728805542, | |
| "rewards/margins_min": 0.15591678023338318, | |
| "rewards/margins_std": 0.3286438286304474, | |
| "rewards/rejected": -0.07015376538038254, | |
| "step": 780 | |
| }, | |
| { | |
| "dpo_losses": 0.4863054156303406, | |
| "epoch": 2.23, | |
| "grad_norm": 3.0491196409319192, | |
| "learning_rate": 9.495303651204494e-08, | |
| "logits/chosen": -2.697542667388916, | |
| "logits/rejected": -2.644935131072998, | |
| "logps/chosen": -334.7068786621094, | |
| "logps/rejected": -294.2601318359375, | |
| "loss": 0.4865, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3127191960811615, | |
| "rewards/margins": 0.4877268671989441, | |
| "rewards/margins_max": 0.660971999168396, | |
| "rewards/margins_min": 0.3144817650318146, | |
| "rewards/margins_std": 0.24500557780265808, | |
| "rewards/rejected": -0.17500770092010498, | |
| "step": 790 | |
| }, | |
| { | |
| "dpo_losses": 0.4887468218803406, | |
| "epoch": 2.25, | |
| "grad_norm": 10.205242036662113, | |
| "learning_rate": 8.860635805202615e-08, | |
| "logits/chosen": -2.723806381225586, | |
| "logits/rejected": -2.6369194984436035, | |
| "logps/chosen": -329.61749267578125, | |
| "logps/rejected": -261.149169921875, | |
| "loss": 0.488, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.33780962228775024, | |
| "rewards/margins": 0.49595513939857483, | |
| "rewards/margins_max": 0.7063379287719727, | |
| "rewards/margins_min": 0.2855724096298218, | |
| "rewards/margins_std": 0.29752615094184875, | |
| "rewards/rejected": -0.15814556181430817, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 2.25, | |
| "eval_dpo_losses": 0.6669806838035583, | |
| "eval_logits/chosen": -2.707551956176758, | |
| "eval_logits/rejected": -2.664377450942993, | |
| "eval_logps/chosen": -288.50439453125, | |
| "eval_logps/rejected": -268.966552734375, | |
| "eval_loss": 1.4895577430725098, | |
| "eval_positive_losses": 8.096439361572266, | |
| "eval_rewards/accuracies": 0.6111111044883728, | |
| "eval_rewards/chosen": -0.03283155709505081, | |
| "eval_rewards/margins": 0.06500754505395889, | |
| "eval_rewards/margins_max": 0.30630454421043396, | |
| "eval_rewards/margins_min": -0.15107154846191406, | |
| "eval_rewards/margins_std": 0.20374149084091187, | |
| "eval_rewards/rejected": -0.09783907979726791, | |
| "eval_runtime": 342.0756, | |
| "eval_samples_per_second": 5.847, | |
| "eval_steps_per_second": 0.184, | |
| "step": 800 | |
| }, | |
| { | |
| "dpo_losses": 0.5151875019073486, | |
| "epoch": 2.28, | |
| "grad_norm": 8.219325520486382, | |
| "learning_rate": 8.24332262395994e-08, | |
| "logits/chosen": -2.7791354656219482, | |
| "logits/rejected": -2.720329999923706, | |
| "logps/chosen": -268.9266052246094, | |
| "logps/rejected": -261.6560363769531, | |
| "loss": 0.4995, | |
| "positive_losses": 0.07818031311035156, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.25640755891799927, | |
| "rewards/margins": 0.4340863823890686, | |
| "rewards/margins_max": 0.7111722230911255, | |
| "rewards/margins_min": 0.15700046718120575, | |
| "rewards/margins_std": 0.39185863733291626, | |
| "rewards/rejected": -0.17767879366874695, | |
| "step": 810 | |
| }, | |
| { | |
| "dpo_losses": 0.5199899673461914, | |
| "epoch": 2.31, | |
| "grad_norm": 11.3286779229284, | |
| "learning_rate": 7.644027904586586e-08, | |
| "logits/chosen": -2.7511487007141113, | |
| "logits/rejected": -2.6695072650909424, | |
| "logps/chosen": -276.63726806640625, | |
| "logps/rejected": -221.2215118408203, | |
| "loss": 0.5014, | |
| "positive_losses": 0.24863624572753906, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2888849079608917, | |
| "rewards/margins": 0.42505329847335815, | |
| "rewards/margins_max": 0.6548727750778198, | |
| "rewards/margins_min": 0.1952337622642517, | |
| "rewards/margins_std": 0.3250138759613037, | |
| "rewards/rejected": -0.13616837561130524, | |
| "step": 820 | |
| }, | |
| { | |
| "dpo_losses": 0.4646497666835785, | |
| "epoch": 2.34, | |
| "grad_norm": 8.714610877659624, | |
| "learning_rate": 7.063396068933469e-08, | |
| "logits/chosen": -2.73659086227417, | |
| "logits/rejected": -2.634260654449463, | |
| "logps/chosen": -415.6689453125, | |
| "logps/rejected": -272.5089111328125, | |
| "loss": 0.4872, | |
| "positive_losses": 0.20364531874656677, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.38227415084838867, | |
| "rewards/margins": 0.5651294589042664, | |
| "rewards/margins_max": 0.7014433145523071, | |
| "rewards/margins_min": 0.4288156032562256, | |
| "rewards/margins_std": 0.19277691841125488, | |
| "rewards/rejected": -0.1828552931547165, | |
| "step": 830 | |
| }, | |
| { | |
| "dpo_losses": 0.49954962730407715, | |
| "epoch": 2.37, | |
| "grad_norm": 6.5716343841546685, | |
| "learning_rate": 6.502051470645148e-08, | |
| "logits/chosen": -2.883086919784546, | |
| "logits/rejected": -2.6989049911499023, | |
| "logps/chosen": -359.88653564453125, | |
| "logps/rejected": -300.28302001953125, | |
| "loss": 0.5152, | |
| "positive_losses": 0.010610580444335938, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.3157649636268616, | |
| "rewards/margins": 0.4664032459259033, | |
| "rewards/margins_max": 0.6633520126342773, | |
| "rewards/margins_min": 0.2694544792175293, | |
| "rewards/margins_std": 0.2785276472568512, | |
| "rewards/rejected": -0.15063826739788055, | |
| "step": 840 | |
| }, | |
| { | |
| "dpo_losses": 0.4566032290458679, | |
| "epoch": 2.39, | |
| "grad_norm": 8.455170840404687, | |
| "learning_rate": 5.960597723792194e-08, | |
| "logits/chosen": -2.742910623550415, | |
| "logits/rejected": -2.619158983230591, | |
| "logps/chosen": -344.3529052734375, | |
| "logps/rejected": -276.9927673339844, | |
| "loss": 0.4916, | |
| "positive_losses": 0.32693710923194885, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.40712490677833557, | |
| "rewards/margins": 0.5964423418045044, | |
| "rewards/margins_max": 0.88237464427948, | |
| "rewards/margins_min": 0.31051012873649597, | |
| "rewards/margins_std": 0.4043692648410797, | |
| "rewards/rejected": -0.1893174648284912, | |
| "step": 850 | |
| }, | |
| { | |
| "dpo_losses": 0.4746529459953308, | |
| "epoch": 2.42, | |
| "grad_norm": 11.331305453316807, | |
| "learning_rate": 5.4396170538046486e-08, | |
| "logits/chosen": -2.788321018218994, | |
| "logits/rejected": -2.7099854946136475, | |
| "logps/chosen": -321.00872802734375, | |
| "logps/rejected": -288.5102844238281, | |
| "loss": 0.4835, | |
| "positive_losses": 0.03827323764562607, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.34411686658859253, | |
| "rewards/margins": 0.5461706519126892, | |
| "rewards/margins_max": 0.8038791418075562, | |
| "rewards/margins_min": 0.28846222162246704, | |
| "rewards/margins_std": 0.36445480585098267, | |
| "rewards/rejected": -0.20205385982990265, | |
| "step": 860 | |
| }, | |
| { | |
| "dpo_losses": 0.515887439250946, | |
| "epoch": 2.45, | |
| "grad_norm": 2.8323654503779396, | |
| "learning_rate": 4.93966967140487e-08, | |
| "logits/chosen": -2.714071750640869, | |
| "logits/rejected": -2.624173641204834, | |
| "logps/chosen": -314.1236267089844, | |
| "logps/rejected": -330.68817138671875, | |
| "loss": 0.4782, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2643023431301117, | |
| "rewards/margins": 0.4125981330871582, | |
| "rewards/margins_max": 0.6127610802650452, | |
| "rewards/margins_min": 0.21243515610694885, | |
| "rewards/margins_std": 0.28307315707206726, | |
| "rewards/rejected": -0.14829573035240173, | |
| "step": 870 | |
| }, | |
| { | |
| "dpo_losses": 0.48759278655052185, | |
| "epoch": 2.48, | |
| "grad_norm": 2.371240249959621, | |
| "learning_rate": 4.4612931702126433e-08, | |
| "logits/chosen": -2.7775819301605225, | |
| "logits/rejected": -2.6954903602600098, | |
| "logps/chosen": -286.12994384765625, | |
| "logps/rejected": -263.48614501953125, | |
| "loss": 0.4882, | |
| "positive_losses": 0.3580247759819031, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.31085458397865295, | |
| "rewards/margins": 0.4919183850288391, | |
| "rewards/margins_max": 0.6966783404350281, | |
| "rewards/margins_min": 0.2871583104133606, | |
| "rewards/margins_std": 0.28957444429397583, | |
| "rewards/rejected": -0.181063711643219, | |
| "step": 880 | |
| }, | |
| { | |
| "dpo_losses": 0.4207460284233093, | |
| "epoch": 2.51, | |
| "grad_norm": 15.410592912064324, | |
| "learning_rate": 4.005001948670605e-08, | |
| "logits/chosen": -2.787081003189087, | |
| "logits/rejected": -2.7058584690093994, | |
| "logps/chosen": -433.3082580566406, | |
| "logps/rejected": -349.69342041015625, | |
| "loss": 0.4854, | |
| "positive_losses": 0.38000088930130005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.43636050820350647, | |
| "rewards/margins": 0.6726460456848145, | |
| "rewards/margins_max": 0.818884015083313, | |
| "rewards/margins_min": 0.5264080762863159, | |
| "rewards/margins_std": 0.20681175589561462, | |
| "rewards/rejected": -0.23628559708595276, | |
| "step": 890 | |
| }, | |
| { | |
| "dpo_losses": 0.44164472818374634, | |
| "epoch": 2.54, | |
| "grad_norm": 9.649985247190592, | |
| "learning_rate": 3.571286656911376e-08, | |
| "logits/chosen": -2.757315158843994, | |
| "logits/rejected": -2.582063674926758, | |
| "logps/chosen": -362.41473388671875, | |
| "logps/rejected": -289.0292663574219, | |
| "loss": 0.5027, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3792465031147003, | |
| "rewards/margins": 0.6161192655563354, | |
| "rewards/margins_max": 0.8344296216964722, | |
| "rewards/margins_min": 0.3978089690208435, | |
| "rewards/margins_std": 0.30873745679855347, | |
| "rewards/rejected": -0.2368728220462799, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 2.54, | |
| "eval_dpo_losses": 0.6661122441291809, | |
| "eval_logits/chosen": -2.7059361934661865, | |
| "eval_logits/rejected": -2.662884473800659, | |
| "eval_logps/chosen": -289.38092041015625, | |
| "eval_logps/rejected": -270.0925598144531, | |
| "eval_loss": 1.5575090646743774, | |
| "eval_positive_losses": 8.782768249511719, | |
| "eval_rewards/accuracies": 0.6190476417541504, | |
| "eval_rewards/chosen": -0.04159707948565483, | |
| "eval_rewards/margins": 0.06750191748142242, | |
| "eval_rewards/margins_max": 0.31507408618927, | |
| "eval_rewards/margins_min": -0.1562977135181427, | |
| "eval_rewards/margins_std": 0.2099103033542633, | |
| "eval_rewards/rejected": -0.10909900069236755, | |
| "eval_runtime": 373.4993, | |
| "eval_samples_per_second": 5.355, | |
| "eval_steps_per_second": 0.169, | |
| "step": 900 | |
| }, | |
| { | |
| "dpo_losses": 0.45572906732559204, | |
| "epoch": 2.56, | |
| "grad_norm": 2.7671497689189013, | |
| "learning_rate": 3.160613669161255e-08, | |
| "logits/chosen": -2.8252711296081543, | |
| "logits/rejected": -2.692148208618164, | |
| "logps/chosen": -382.2273254394531, | |
| "logps/rejected": -260.42022705078125, | |
| "loss": 0.4747, | |
| "positive_losses": 0.24142150580883026, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.36954694986343384, | |
| "rewards/margins": 0.5896551012992859, | |
| "rewards/margins_max": 0.8211862444877625, | |
| "rewards/margins_min": 0.35812389850616455, | |
| "rewards/margins_std": 0.32743456959724426, | |
| "rewards/rejected": -0.22010818123817444, | |
| "step": 910 | |
| }, | |
| { | |
| "dpo_losses": 0.4458266794681549, | |
| "epoch": 2.59, | |
| "grad_norm": 2.5946275076947987, | |
| "learning_rate": 2.7734245822478436e-08, | |
| "logits/chosen": -2.7092032432556152, | |
| "logits/rejected": -2.5538458824157715, | |
| "logps/chosen": -320.76727294921875, | |
| "logps/rejected": -214.5659942626953, | |
| "loss": 0.4918, | |
| "positive_losses": 0.05587196350097656, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.4010286331176758, | |
| "rewards/margins": 0.6215327382087708, | |
| "rewards/margins_max": 0.8581794500350952, | |
| "rewards/margins_min": 0.3848857879638672, | |
| "rewards/margins_std": 0.3346691429615021, | |
| "rewards/rejected": -0.2205040454864502, | |
| "step": 920 | |
| }, | |
| { | |
| "dpo_losses": 0.48411306738853455, | |
| "epoch": 2.62, | |
| "grad_norm": 9.968121643352859, | |
| "learning_rate": 2.410135740750821e-08, | |
| "logits/chosen": -2.7608590126037598, | |
| "logits/rejected": -2.713438034057617, | |
| "logps/chosen": -317.3198547363281, | |
| "logps/rejected": -291.6824645996094, | |
| "loss": 0.4877, | |
| "positive_losses": 0.46479111909866333, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2928626835346222, | |
| "rewards/margins": 0.5131780505180359, | |
| "rewards/margins_max": 0.6953937411308289, | |
| "rewards/margins_min": 0.33096247911453247, | |
| "rewards/margins_std": 0.25769174098968506, | |
| "rewards/rejected": -0.22031545639038086, | |
| "step": 930 | |
| }, | |
| { | |
| "dpo_losses": 0.5361374616622925, | |
| "epoch": 2.65, | |
| "grad_norm": 13.584990105876331, | |
| "learning_rate": 2.071137789306418e-08, | |
| "logits/chosen": -2.739955425262451, | |
| "logits/rejected": -2.62253999710083, | |
| "logps/chosen": -313.5608825683594, | |
| "logps/rejected": -221.56198120117188, | |
| "loss": 0.499, | |
| "positive_losses": 0.22314663231372833, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2565317451953888, | |
| "rewards/margins": 0.3702241778373718, | |
| "rewards/margins_max": 0.5549638271331787, | |
| "rewards/margins_min": 0.18548452854156494, | |
| "rewards/margins_std": 0.26126131415367126, | |
| "rewards/rejected": -0.11369242519140244, | |
| "step": 940 | |
| }, | |
| { | |
| "dpo_losses": 0.4967280328273773, | |
| "epoch": 2.68, | |
| "grad_norm": 8.2589469715046, | |
| "learning_rate": 1.7567952525471107e-08, | |
| "logits/chosen": -2.7256572246551514, | |
| "logits/rejected": -2.615950584411621, | |
| "logps/chosen": -271.1017150878906, | |
| "logps/rejected": -220.56100463867188, | |
| "loss": 0.5302, | |
| "positive_losses": 1.0882877111434937, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.31043368577957153, | |
| "rewards/margins": 0.48788315057754517, | |
| "rewards/margins_max": 0.704791784286499, | |
| "rewards/margins_min": 0.2709745466709137, | |
| "rewards/margins_std": 0.30675509572029114, | |
| "rewards/rejected": -0.17744943499565125, | |
| "step": 950 | |
| }, | |
| { | |
| "dpo_losses": 0.4955645501613617, | |
| "epoch": 2.7, | |
| "grad_norm": 18.114348971792364, | |
| "learning_rate": 1.467446143128101e-08, | |
| "logits/chosen": -2.8821053504943848, | |
| "logits/rejected": -2.793936252593994, | |
| "logps/chosen": -297.2878112792969, | |
| "logps/rejected": -261.58392333984375, | |
| "loss": 0.5139, | |
| "positive_losses": 0.06438522040843964, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.29606151580810547, | |
| "rewards/margins": 0.4726499915122986, | |
| "rewards/margins_max": 0.6590073704719543, | |
| "rewards/margins_min": 0.2862926125526428, | |
| "rewards/margins_std": 0.26354914903640747, | |
| "rewards/rejected": -0.1765884906053543, | |
| "step": 960 | |
| }, | |
| { | |
| "dpo_losses": 0.47625431418418884, | |
| "epoch": 2.73, | |
| "grad_norm": 2.2745052596169986, | |
| "learning_rate": 1.2034015982622243e-08, | |
| "logits/chosen": -2.7958290576934814, | |
| "logits/rejected": -2.691840887069702, | |
| "logps/chosen": -341.50140380859375, | |
| "logps/rejected": -322.9205627441406, | |
| "loss": 0.4943, | |
| "positive_losses": 0.23634567856788635, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.33635348081588745, | |
| "rewards/margins": 0.5307850241661072, | |
| "rewards/margins_max": 0.7724507451057434, | |
| "rewards/margins_min": 0.2891193926334381, | |
| "rewards/margins_std": 0.34176692366600037, | |
| "rewards/rejected": -0.1944316029548645, | |
| "step": 970 | |
| }, | |
| { | |
| "dpo_losses": 0.514926552772522, | |
| "epoch": 2.76, | |
| "grad_norm": 9.098303215376248, | |
| "learning_rate": 9.649455451539418e-09, | |
| "logits/chosen": -2.6313929557800293, | |
| "logits/rejected": -2.6137583255767822, | |
| "logps/chosen": -211.71536254882812, | |
| "logps/rejected": -190.9324493408203, | |
| "loss": 0.4996, | |
| "positive_losses": 0.12087974697351456, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2592369318008423, | |
| "rewards/margins": 0.4249873757362366, | |
| "rewards/margins_max": 0.6153701543807983, | |
| "rewards/margins_min": 0.2346045970916748, | |
| "rewards/margins_std": 0.26924189925193787, | |
| "rewards/rejected": -0.1657504439353943, | |
| "step": 980 | |
| }, | |
| { | |
| "dpo_losses": 0.471360445022583, | |
| "epoch": 2.79, | |
| "grad_norm": 7.946144983120647, | |
| "learning_rate": 7.523343956923194e-09, | |
| "logits/chosen": -2.797616481781006, | |
| "logits/rejected": -2.7332680225372314, | |
| "logps/chosen": -315.5093688964844, | |
| "logps/rejected": -304.4640808105469, | |
| "loss": 0.4919, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.36568814516067505, | |
| "rewards/margins": 0.5568908452987671, | |
| "rewards/margins_max": 0.824783980846405, | |
| "rewards/margins_min": 0.28899770975112915, | |
| "rewards/margins_std": 0.37885811924934387, | |
| "rewards/rejected": -0.19120268523693085, | |
| "step": 990 | |
| }, | |
| { | |
| "dpo_losses": 0.513391375541687, | |
| "epoch": 2.82, | |
| "grad_norm": 7.57336855401101, | |
| "learning_rate": 5.6579677073121945e-09, | |
| "logits/chosen": -2.6762197017669678, | |
| "logits/rejected": -2.677922487258911, | |
| "logps/chosen": -255.44204711914062, | |
| "logps/rejected": -303.0498962402344, | |
| "loss": 0.4962, | |
| "positive_losses": 0.01267166156321764, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2648389935493469, | |
| "rewards/margins": 0.43319040536880493, | |
| "rewards/margins_max": 0.6835809946060181, | |
| "rewards/margins_min": 0.18279966711997986, | |
| "rewards/margins_std": 0.3541058897972107, | |
| "rewards/rejected": -0.168351411819458, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 2.82, | |
| "eval_dpo_losses": 0.6659845113754272, | |
| "eval_logits/chosen": -2.7036855220794678, | |
| "eval_logits/rejected": -2.6605641841888428, | |
| "eval_logps/chosen": -289.52734375, | |
| "eval_logps/rejected": -270.2825012207031, | |
| "eval_loss": 1.5707319974899292, | |
| "eval_positive_losses": 8.908056259155273, | |
| "eval_rewards/accuracies": 0.6150793433189392, | |
| "eval_rewards/chosen": -0.04306148737668991, | |
| "eval_rewards/margins": 0.06793692708015442, | |
| "eval_rewards/margins_max": 0.3166824281215668, | |
| "eval_rewards/margins_min": -0.15675365924835205, | |
| "eval_rewards/margins_std": 0.2111140936613083, | |
| "eval_rewards/rejected": -0.11099842935800552, | |
| "eval_runtime": 385.4819, | |
| "eval_samples_per_second": 5.188, | |
| "eval_steps_per_second": 0.163, | |
| "step": 1000 | |
| }, | |
| { | |
| "dpo_losses": 0.4899842143058777, | |
| "epoch": 2.85, | |
| "grad_norm": 11.596233256209617, | |
| "learning_rate": 4.0553325425319585e-09, | |
| "logits/chosen": -2.7825233936309814, | |
| "logits/rejected": -2.710404872894287, | |
| "logps/chosen": -283.56024169921875, | |
| "logps/rejected": -277.7485656738281, | |
| "loss": 0.5045, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.32106488943099976, | |
| "rewards/margins": 0.503676176071167, | |
| "rewards/margins_max": 0.7390400171279907, | |
| "rewards/margins_min": 0.2683122456073761, | |
| "rewards/margins_std": 0.3328548073768616, | |
| "rewards/rejected": -0.18261122703552246, | |
| "step": 1010 | |
| }, | |
| { | |
| "dpo_losses": 0.4729464650154114, | |
| "epoch": 2.87, | |
| "grad_norm": 2.4410647222056254, | |
| "learning_rate": 2.717161776814747e-09, | |
| "logits/chosen": -2.742475986480713, | |
| "logits/rejected": -2.64007568359375, | |
| "logps/chosen": -277.25311279296875, | |
| "logps/rejected": -253.4837188720703, | |
| "loss": 0.4843, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.3422829210758209, | |
| "rewards/margins": 0.5467172861099243, | |
| "rewards/margins_max": 0.7624825835227966, | |
| "rewards/margins_min": 0.3309520483016968, | |
| "rewards/margins_std": 0.30513814091682434, | |
| "rewards/rejected": -0.20443439483642578, | |
| "step": 1020 | |
| }, | |
| { | |
| "dpo_losses": 0.48559775948524475, | |
| "epoch": 2.9, | |
| "grad_norm": 8.212943342949115, | |
| "learning_rate": 1.6448943457189613e-09, | |
| "logits/chosen": -2.765453815460205, | |
| "logits/rejected": -2.705345630645752, | |
| "logps/chosen": -331.9368896484375, | |
| "logps/rejected": -294.2420349121094, | |
| "loss": 0.4995, | |
| "positive_losses": 0.18167057633399963, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.33231136202812195, | |
| "rewards/margins": 0.5095351934432983, | |
| "rewards/margins_max": 0.6868753433227539, | |
| "rewards/margins_min": 0.332194983959198, | |
| "rewards/margins_std": 0.2507968842983246, | |
| "rewards/rejected": -0.17722377181053162, | |
| "step": 1030 | |
| }, | |
| { | |
| "dpo_losses": 0.4621458053588867, | |
| "epoch": 2.93, | |
| "grad_norm": 8.550282024171548, | |
| "learning_rate": 8.396832588411229e-10, | |
| "logits/chosen": -2.6322901248931885, | |
| "logits/rejected": -2.517449378967285, | |
| "logps/chosen": -312.41729736328125, | |
| "logps/rejected": -251.61849975585938, | |
| "loss": 0.5147, | |
| "positive_losses": 0.4247921109199524, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.33802804350852966, | |
| "rewards/margins": 0.5674580335617065, | |
| "rewards/margins_max": 0.8604093790054321, | |
| "rewards/margins_min": 0.2745068669319153, | |
| "rewards/margins_std": 0.41429558396339417, | |
| "rewards/rejected": -0.22943000495433807, | |
| "step": 1040 | |
| }, | |
| { | |
| "dpo_losses": 0.49349918961524963, | |
| "epoch": 2.96, | |
| "grad_norm": 2.0362116151192713, | |
| "learning_rate": 3.0239435998430374e-10, | |
| "logits/chosen": -2.7547783851623535, | |
| "logits/rejected": -2.6397135257720947, | |
| "logps/chosen": -298.6687927246094, | |
| "logps/rejected": -274.22381591796875, | |
| "loss": 0.5095, | |
| "positive_losses": 0.5305103063583374, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.32915833592414856, | |
| "rewards/margins": 0.4934717118740082, | |
| "rewards/margins_max": 0.7064648866653442, | |
| "rewards/margins_min": 0.28047865629196167, | |
| "rewards/margins_std": 0.3012176752090454, | |
| "rewards/rejected": -0.1643133908510208, | |
| "step": 1050 | |
| }, | |
| { | |
| "dpo_losses": 0.5019891858100891, | |
| "epoch": 2.99, | |
| "grad_norm": 2.0527644568071164, | |
| "learning_rate": 3.360539611582669e-11, | |
| "logits/chosen": -2.6836140155792236, | |
| "logits/rejected": -2.597043514251709, | |
| "logps/chosen": -294.3243103027344, | |
| "logps/rejected": -225.4201202392578, | |
| "loss": 0.5343, | |
| "positive_losses": 0.3622688353061676, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.3153269588947296, | |
| "rewards/margins": 0.4751824736595154, | |
| "rewards/margins_max": 0.7748254537582397, | |
| "rewards/margins_min": 0.1755395084619522, | |
| "rewards/margins_std": 0.4237591624259949, | |
| "rewards/rejected": -0.15985555946826935, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "step": 1065, | |
| "total_flos": 0.0, | |
| "train_loss": 0.5596254680078354, | |
| "train_runtime": 11165.5936, | |
| "train_samples_per_second": 1.526, | |
| "train_steps_per_second": 0.095 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1065, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |