| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.998691442030882, |
| "eval_steps": 50, |
| "global_step": 477, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "dpo_lambda": 1.0, |
| "epoch": 0.002093692750588851, |
| "grad_norm": 4.402319572917802, |
| "learning_rate": 1.0416666666666666e-08, |
| "logits/chosen": -0.8043479323387146, |
| "logits/rejected": -0.8551070690155029, |
| "logps/chosen": -318.6319885253906, |
| "logps/rejected": -337.8906555175781, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "dpo_lambda": 0.9998951554298401, |
| "epoch": 0.004187385501177702, |
| "grad_norm": 4.793462944453188, |
| "learning_rate": 2.083333333333333e-08, |
| "logits/chosen": -0.7836206555366516, |
| "logits/rejected": -0.9540650844573975, |
| "logps/chosen": -330.71966552734375, |
| "logps/rejected": -286.4294128417969, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 2 |
| }, |
| { |
| "dpo_lambda": 0.9997903108596802, |
| "epoch": 0.006281078251766554, |
| "grad_norm": 5.083035763031515, |
| "learning_rate": 3.125e-08, |
| "logits/chosen": -0.8143987655639648, |
| "logits/rejected": -0.8199301958084106, |
| "logps/chosen": -276.8949890136719, |
| "logps/rejected": -268.3603820800781, |
| "loss": 0.6932, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.0008645334746688604, |
| "rewards/margins": -0.001431022654287517, |
| "rewards/rejected": 0.0005664890632033348, |
| "step": 3 |
| }, |
| { |
| "dpo_lambda": 0.9996854066848755, |
| "epoch": 0.008374771002355404, |
| "grad_norm": 5.902572234969009, |
| "learning_rate": 4.166666666666666e-08, |
| "logits/chosen": -0.8467217087745667, |
| "logits/rejected": -0.8571422100067139, |
| "logps/chosen": -300.71966552734375, |
| "logps/rejected": -329.18927001953125, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 6.150537228677422e-05, |
| "rewards/margins": -0.00012183257786091417, |
| "rewards/rejected": 0.00018333786283619702, |
| "step": 4 |
| }, |
| { |
| "dpo_lambda": 0.9995808005332947, |
| "epoch": 0.010468463752944255, |
| "grad_norm": 5.030790216066496, |
| "learning_rate": 5.208333333333333e-08, |
| "logits/chosen": -0.854633092880249, |
| "logits/rejected": -0.8405370116233826, |
| "logps/chosen": -263.6219482421875, |
| "logps/rejected": -256.4416198730469, |
| "loss": 0.693, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.00030443898867815733, |
| "rewards/margins": -0.0004514011961873621, |
| "rewards/rejected": 0.00014696212019771338, |
| "step": 5 |
| }, |
| { |
| "dpo_lambda": 0.99947589635849, |
| "epoch": 0.012562156503533107, |
| "grad_norm": 6.449855736894302, |
| "learning_rate": 6.25e-08, |
| "logits/chosen": -0.848020076751709, |
| "logits/rejected": -0.879482626914978, |
| "logps/chosen": -288.12579345703125, |
| "logps/rejected": -244.55783081054688, |
| "loss": 0.6935, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.0011313408613204956, |
| "rewards/margins": -0.0024054457899183035, |
| "rewards/rejected": 0.0012741051614284515, |
| "step": 6 |
| }, |
| { |
| "dpo_lambda": 0.9993710517883301, |
| "epoch": 0.014655849254121958, |
| "grad_norm": 4.4471205365033715, |
| "learning_rate": 7.291666666666667e-08, |
| "logits/chosen": -0.8126897215843201, |
| "logits/rejected": -0.9293793439865112, |
| "logps/chosen": -359.7424621582031, |
| "logps/rejected": -305.7148742675781, |
| "loss": 0.6926, |
| "rewards/accuracies": 0.515625, |
| "rewards/chosen": 0.00028104332159273326, |
| "rewards/margins": 0.0005955130327492952, |
| "rewards/rejected": -0.00031446965294890106, |
| "step": 7 |
| }, |
| { |
| "dpo_lambda": 0.9992662072181702, |
| "epoch": 0.016749542004710807, |
| "grad_norm": 5.6547119853423276, |
| "learning_rate": 8.333333333333333e-08, |
| "logits/chosen": -0.8336946964263916, |
| "logits/rejected": -0.8718341588973999, |
| "logps/chosen": -305.8148193359375, |
| "logps/rejected": -274.6888732910156, |
| "loss": 0.6935, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.00020592098007909954, |
| "rewards/margins": -0.00026029106811620295, |
| "rewards/rejected": 5.437021900434047e-05, |
| "step": 8 |
| }, |
| { |
| "dpo_lambda": 0.9991613626480103, |
| "epoch": 0.01884323475529966, |
| "grad_norm": 3.9638195143574837, |
| "learning_rate": 9.375e-08, |
| "logits/chosen": -0.7655616402626038, |
| "logits/rejected": -0.908790111541748, |
| "logps/chosen": -326.245361328125, |
| "logps/rejected": -278.6490478515625, |
| "loss": 0.6933, |
| "rewards/accuracies": 0.546875, |
| "rewards/chosen": 0.0012860479764640331, |
| "rewards/margins": 0.000617672863882035, |
| "rewards/rejected": 0.000668375170789659, |
| "step": 9 |
| }, |
| { |
| "dpo_lambda": 0.9990566968917847, |
| "epoch": 0.02093692750588851, |
| "grad_norm": 4.403617727179671, |
| "learning_rate": 1.0416666666666667e-07, |
| "logits/chosen": -0.7897341251373291, |
| "logits/rejected": -0.853586733341217, |
| "logps/chosen": -327.54168701171875, |
| "logps/rejected": -333.5374450683594, |
| "loss": 0.6934, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.00038820982445031404, |
| "rewards/margins": -0.0005907297600060701, |
| "rewards/rejected": 0.0009789395844563842, |
| "step": 10 |
| }, |
| { |
| "dpo_lambda": 0.9989518523216248, |
| "epoch": 0.023030620256477362, |
| "grad_norm": 7.796052749512685, |
| "learning_rate": 1.1458333333333332e-07, |
| "logits/chosen": -0.8637887239456177, |
| "logits/rejected": -0.9242954254150391, |
| "logps/chosen": -261.32867431640625, |
| "logps/rejected": -249.37728881835938, |
| "loss": 0.6933, |
| "rewards/accuracies": 0.546875, |
| "rewards/chosen": -0.00039038294926285744, |
| "rewards/margins": -0.00036161605385132134, |
| "rewards/rejected": -2.8766971809091046e-05, |
| "step": 11 |
| }, |
| { |
| "dpo_lambda": 0.9988469481468201, |
| "epoch": 0.025124313007066214, |
| "grad_norm": 3.8821871111757082, |
| "learning_rate": 1.25e-07, |
| "logits/chosen": -0.9050667881965637, |
| "logits/rejected": -0.8800326585769653, |
| "logps/chosen": -280.99847412109375, |
| "logps/rejected": -275.5593566894531, |
| "loss": 0.6929, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.001003009034320712, |
| "rewards/margins": -0.00038465563557110727, |
| "rewards/rejected": -0.0006183534860610962, |
| "step": 12 |
| }, |
| { |
| "dpo_lambda": 0.9987421631813049, |
| "epoch": 0.027218005757655064, |
| "grad_norm": 6.440110466185814, |
| "learning_rate": 1.3541666666666666e-07, |
| "logits/chosen": -0.826982855796814, |
| "logits/rejected": -0.8742626309394836, |
| "logps/chosen": -280.1139831542969, |
| "logps/rejected": -244.7361297607422, |
| "loss": 0.6932, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": 0.00022524214000441134, |
| "rewards/margins": -0.0009332930203527212, |
| "rewards/rejected": 0.001158535131253302, |
| "step": 13 |
| }, |
| { |
| "dpo_lambda": 0.9986372590065002, |
| "epoch": 0.029311698508243916, |
| "grad_norm": 4.229417019989064, |
| "learning_rate": 1.4583333333333335e-07, |
| "logits/chosen": -0.8535632491111755, |
| "logits/rejected": -0.8801539540290833, |
| "logps/chosen": -328.8179016113281, |
| "logps/rejected": -320.2232971191406, |
| "loss": 0.6937, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.000532336242031306, |
| "rewards/margins": -0.0019397324649617076, |
| "rewards/rejected": 0.0014073961647227407, |
| "step": 14 |
| }, |
| { |
| "dpo_lambda": 0.9985324144363403, |
| "epoch": 0.031405391258832765, |
| "grad_norm": 4.467816070741843, |
| "learning_rate": 1.5624999999999999e-07, |
| "logits/chosen": -0.8298668265342712, |
| "logits/rejected": -0.9168381690979004, |
| "logps/chosen": -303.4217224121094, |
| "logps/rejected": -259.37548828125, |
| "loss": 0.6932, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 6.753446359653026e-05, |
| "rewards/margins": -0.0004937391495332122, |
| "rewards/rejected": 0.0005612736567854881, |
| "step": 15 |
| }, |
| { |
| "dpo_lambda": 0.9984277486801147, |
| "epoch": 0.033499084009421615, |
| "grad_norm": 3.888254222848592, |
| "learning_rate": 1.6666666666666665e-07, |
| "logits/chosen": -0.7577842473983765, |
| "logits/rejected": -0.8261817693710327, |
| "logps/chosen": -297.6632385253906, |
| "logps/rejected": -301.09295654296875, |
| "loss": 0.6929, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 1.160617102868855e-05, |
| "rewards/margins": 0.00021146482322365046, |
| "rewards/rejected": -0.00019985856488347054, |
| "step": 16 |
| }, |
| { |
| "dpo_lambda": 0.9983229041099548, |
| "epoch": 0.03559277676001047, |
| "grad_norm": 4.452768078866186, |
| "learning_rate": 1.7708333333333334e-07, |
| "logits/chosen": -0.868607759475708, |
| "logits/rejected": -0.8844305276870728, |
| "logps/chosen": -289.6638488769531, |
| "logps/rejected": -255.01210021972656, |
| "loss": 0.6932, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": 0.00011004415864590555, |
| "rewards/margins": -0.0005320608615875244, |
| "rewards/rejected": 0.0006421051220968366, |
| "step": 17 |
| }, |
| { |
| "dpo_lambda": 0.9982179999351501, |
| "epoch": 0.03768646951059932, |
| "grad_norm": 4.8480605497744, |
| "learning_rate": 1.875e-07, |
| "logits/chosen": -0.7690554261207581, |
| "logits/rejected": -0.9109585285186768, |
| "logps/chosen": -349.46014404296875, |
| "logps/rejected": -280.4686584472656, |
| "loss": 0.6929, |
| "rewards/accuracies": 0.515625, |
| "rewards/chosen": 0.0018899872666224837, |
| "rewards/margins": 0.001062125200405717, |
| "rewards/rejected": 0.0008278620080091059, |
| "step": 18 |
| }, |
| { |
| "dpo_lambda": 0.998113214969635, |
| "epoch": 0.03978016226118817, |
| "grad_norm": 4.425003512881252, |
| "learning_rate": 1.9791666666666664e-07, |
| "logits/chosen": -0.7577191591262817, |
| "logits/rejected": -0.8322458863258362, |
| "logps/chosen": -324.3868713378906, |
| "logps/rejected": -302.09149169921875, |
| "loss": 0.6927, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.002668360248208046, |
| "rewards/margins": 0.0012416491517797112, |
| "rewards/rejected": 0.0014267113292589784, |
| "step": 19 |
| }, |
| { |
| "dpo_lambda": 0.9980083107948303, |
| "epoch": 0.04187385501177702, |
| "grad_norm": 4.846085404616861, |
| "learning_rate": 2.0833333333333333e-07, |
| "logits/chosen": -0.8820050954818726, |
| "logits/rejected": -0.9251440763473511, |
| "logps/chosen": -268.20587158203125, |
| "logps/rejected": -281.5534973144531, |
| "loss": 0.6929, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.002546853618696332, |
| "rewards/margins": 0.0012899779248982668, |
| "rewards/rejected": 0.001256875810213387, |
| "step": 20 |
| }, |
| { |
| "dpo_lambda": 0.9979037046432495, |
| "epoch": 0.043967547762365874, |
| "grad_norm": 4.117368237143938, |
| "learning_rate": 2.1875e-07, |
| "logits/chosen": -0.8503252267837524, |
| "logits/rejected": -0.92568039894104, |
| "logps/chosen": -226.6487274169922, |
| "logps/rejected": -226.0305633544922, |
| "loss": 0.6927, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": 0.002446984639391303, |
| "rewards/margins": 0.0014305507065728307, |
| "rewards/rejected": 0.0010164338164031506, |
| "step": 21 |
| }, |
| { |
| "dpo_lambda": 0.9977988004684448, |
| "epoch": 0.046061240512954724, |
| "grad_norm": 5.26507499436042, |
| "learning_rate": 2.2916666666666663e-07, |
| "logits/chosen": -0.7547565698623657, |
| "logits/rejected": -0.7918416261672974, |
| "logps/chosen": -299.7794189453125, |
| "logps/rejected": -281.09222412109375, |
| "loss": 0.6919, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.00438675656914711, |
| "rewards/margins": 0.003029727144166827, |
| "rewards/rejected": 0.0013570297742262483, |
| "step": 22 |
| }, |
| { |
| "dpo_lambda": 0.9976939558982849, |
| "epoch": 0.04815493326354357, |
| "grad_norm": 4.528414091774314, |
| "learning_rate": 2.3958333333333335e-07, |
| "logits/chosen": -0.8128759860992432, |
| "logits/rejected": -0.8892397284507751, |
| "logps/chosen": -342.2197265625, |
| "logps/rejected": -282.14398193359375, |
| "loss": 0.6922, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": 0.0038836237508803606, |
| "rewards/margins": 0.001989413285627961, |
| "rewards/rejected": 0.0018942105816677213, |
| "step": 23 |
| }, |
| { |
| "dpo_lambda": 0.997589111328125, |
| "epoch": 0.05024862601413243, |
| "grad_norm": 4.040981518845972, |
| "learning_rate": 2.5e-07, |
| "logits/chosen": -0.743541955947876, |
| "logits/rejected": -0.9808051586151123, |
| "logps/chosen": -350.8001403808594, |
| "logps/rejected": -279.0107727050781, |
| "loss": 0.6921, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": 0.005917892791330814, |
| "rewards/margins": 0.002584913745522499, |
| "rewards/rejected": 0.003332979278638959, |
| "step": 24 |
| }, |
| { |
| "dpo_lambda": 0.9974842667579651, |
| "epoch": 0.05234231876472128, |
| "grad_norm": 4.923955440464468, |
| "learning_rate": 2.604166666666667e-07, |
| "logits/chosen": -0.7823559045791626, |
| "logits/rejected": -0.9089896082878113, |
| "logps/chosen": -280.17523193359375, |
| "logps/rejected": -243.14208984375, |
| "loss": 0.6921, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.004923699423670769, |
| "rewards/margins": 0.0027135859709233046, |
| "rewards/rejected": 0.002210113452747464, |
| "step": 25 |
| }, |
| { |
| "dpo_lambda": 0.9973794221878052, |
| "epoch": 0.05443601151531013, |
| "grad_norm": 4.351217149891559, |
| "learning_rate": 2.708333333333333e-07, |
| "logits/chosen": -0.7708956003189087, |
| "logits/rejected": -0.9697441458702087, |
| "logps/chosen": -338.752197265625, |
| "logps/rejected": -289.9631652832031, |
| "loss": 0.6915, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.009810867719352245, |
| "rewards/margins": 0.004754879977554083, |
| "rewards/rejected": 0.005055988673120737, |
| "step": 26 |
| }, |
| { |
| "dpo_lambda": 0.9972745180130005, |
| "epoch": 0.056529704265898977, |
| "grad_norm": 4.473346542639472, |
| "learning_rate": 2.8125e-07, |
| "logits/chosen": -0.8319710493087769, |
| "logits/rejected": -0.9578927755355835, |
| "logps/chosen": -320.4068908691406, |
| "logps/rejected": -257.923583984375, |
| "loss": 0.692, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.009162155911326408, |
| "rewards/margins": 0.0028106458485126495, |
| "rewards/rejected": 0.006351510062813759, |
| "step": 27 |
| }, |
| { |
| "dpo_lambda": 0.9971698522567749, |
| "epoch": 0.05862339701648783, |
| "grad_norm": 4.3426457588367136, |
| "learning_rate": 2.916666666666667e-07, |
| "logits/chosen": -0.7956329584121704, |
| "logits/rejected": -0.8427987694740295, |
| "logps/chosen": -320.89215087890625, |
| "logps/rejected": -300.1125793457031, |
| "loss": 0.6913, |
| "rewards/accuracies": 0.546875, |
| "rewards/chosen": 0.01058600191026926, |
| "rewards/margins": 0.00340261054225266, |
| "rewards/rejected": 0.007183392066508532, |
| "step": 28 |
| }, |
| { |
| "dpo_lambda": 0.997065007686615, |
| "epoch": 0.06071708976707668, |
| "grad_norm": 4.085431965259117, |
| "learning_rate": 3.020833333333333e-07, |
| "logits/chosen": -0.726355254650116, |
| "logits/rejected": -0.9321560859680176, |
| "logps/chosen": -381.36444091796875, |
| "logps/rejected": -298.1344299316406, |
| "loss": 0.6919, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": 0.013005386106669903, |
| "rewards/margins": 0.00456521799787879, |
| "rewards/rejected": 0.0084401685744524, |
| "step": 29 |
| }, |
| { |
| "dpo_lambda": 0.9969601631164551, |
| "epoch": 0.06281078251766553, |
| "grad_norm": 3.783478029121267, |
| "learning_rate": 3.1249999999999997e-07, |
| "logits/chosen": -0.790072500705719, |
| "logits/rejected": -0.8063573837280273, |
| "logps/chosen": -345.5115661621094, |
| "logps/rejected": -364.02142333984375, |
| "loss": 0.6911, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": 0.014460022561252117, |
| "rewards/margins": 0.002242105081677437, |
| "rewards/rejected": 0.012217918410897255, |
| "step": 30 |
| }, |
| { |
| "dpo_lambda": 0.9968553185462952, |
| "epoch": 0.06490447526825438, |
| "grad_norm": 4.725066162809079, |
| "learning_rate": 3.2291666666666666e-07, |
| "logits/chosen": -0.7989029884338379, |
| "logits/rejected": -0.8742246031761169, |
| "logps/chosen": -305.10986328125, |
| "logps/rejected": -267.1070251464844, |
| "loss": 0.6915, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": 0.015996551141142845, |
| "rewards/margins": 0.004079596605151892, |
| "rewards/rejected": 0.011916955932974815, |
| "step": 31 |
| }, |
| { |
| "dpo_lambda": 0.9967504739761353, |
| "epoch": 0.06699816801884323, |
| "grad_norm": 6.263003607437198, |
| "learning_rate": 3.333333333333333e-07, |
| "logits/chosen": -0.7864735126495361, |
| "logits/rejected": -0.7694317102432251, |
| "logps/chosen": -314.7187194824219, |
| "logps/rejected": -318.4085693359375, |
| "loss": 0.6904, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": 0.018498174846172333, |
| "rewards/margins": 0.00419366592541337, |
| "rewards/rejected": 0.01430450938642025, |
| "step": 32 |
| }, |
| { |
| "dpo_lambda": 0.9966458082199097, |
| "epoch": 0.06909186076943209, |
| "grad_norm": 4.996565408773174, |
| "learning_rate": 3.4375e-07, |
| "logits/chosen": -0.8627775311470032, |
| "logits/rejected": -0.9619947075843811, |
| "logps/chosen": -329.125732421875, |
| "logps/rejected": -290.11798095703125, |
| "loss": 0.689, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.022791242226958275, |
| "rewards/margins": 0.008790804073214531, |
| "rewards/rejected": 0.01400043722242117, |
| "step": 33 |
| }, |
| { |
| "dpo_lambda": 0.9965409636497498, |
| "epoch": 0.07118555352002094, |
| "grad_norm": 4.077713925438457, |
| "learning_rate": 3.541666666666667e-07, |
| "logits/chosen": -0.8728955984115601, |
| "logits/rejected": -0.9118562340736389, |
| "logps/chosen": -257.28759765625, |
| "logps/rejected": -223.10824584960938, |
| "loss": 0.6892, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.02388455532491207, |
| "rewards/margins": 0.009245344437658787, |
| "rewards/rejected": 0.01463920809328556, |
| "step": 34 |
| }, |
| { |
| "dpo_lambda": 0.9964360594749451, |
| "epoch": 0.07327924627060979, |
| "grad_norm": 4.163606742901362, |
| "learning_rate": 3.645833333333333e-07, |
| "logits/chosen": -0.8726727366447449, |
| "logits/rejected": -0.9065307378768921, |
| "logps/chosen": -219.03260803222656, |
| "logps/rejected": -223.0487518310547, |
| "loss": 0.6897, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.023589186370372772, |
| "rewards/margins": 0.004913577809929848, |
| "rewards/rejected": 0.018675606697797775, |
| "step": 35 |
| }, |
| { |
| "dpo_lambda": 0.9963312149047852, |
| "epoch": 0.07537293902119864, |
| "grad_norm": 4.814967779155049, |
| "learning_rate": 3.75e-07, |
| "logits/chosen": -0.7583918571472168, |
| "logits/rejected": -0.8758710622787476, |
| "logps/chosen": -299.6081848144531, |
| "logps/rejected": -260.1003723144531, |
| "loss": 0.689, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.030413059517741203, |
| "rewards/margins": 0.009431293234229088, |
| "rewards/rejected": 0.020981768146157265, |
| "step": 36 |
| }, |
| { |
| "dpo_lambda": 0.9962263703346252, |
| "epoch": 0.07746663177178749, |
| "grad_norm": 6.11369907380103, |
| "learning_rate": 3.8541666666666665e-07, |
| "logits/chosen": -0.8049490451812744, |
| "logits/rejected": -0.8439275622367859, |
| "logps/chosen": -332.357421875, |
| "logps/rejected": -262.38671875, |
| "loss": 0.6875, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.036942947655916214, |
| "rewards/margins": 0.01394661981612444, |
| "rewards/rejected": 0.0229963306337595, |
| "step": 37 |
| }, |
| { |
| "dpo_lambda": 0.9961215257644653, |
| "epoch": 0.07956032452237634, |
| "grad_norm": 4.137117122207877, |
| "learning_rate": 3.958333333333333e-07, |
| "logits/chosen": -0.8226761817932129, |
| "logits/rejected": -0.8536137342453003, |
| "logps/chosen": -258.0992126464844, |
| "logps/rejected": -246.155029296875, |
| "loss": 0.687, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": 0.03529813885688782, |
| "rewards/margins": 0.0135354558005929, |
| "rewards/rejected": 0.021762683987617493, |
| "step": 38 |
| }, |
| { |
| "dpo_lambda": 0.9960166215896606, |
| "epoch": 0.08165401727296519, |
| "grad_norm": 4.21153020487944, |
| "learning_rate": 4.0625e-07, |
| "logits/chosen": -0.7359837293624878, |
| "logits/rejected": -0.8013171553611755, |
| "logps/chosen": -321.1292724609375, |
| "logps/rejected": -266.85919189453125, |
| "loss": 0.6849, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.040478628128767014, |
| "rewards/margins": 0.014545298181474209, |
| "rewards/rejected": 0.02593333274126053, |
| "step": 39 |
| }, |
| { |
| "dpo_lambda": 0.9959120154380798, |
| "epoch": 0.08374771002355404, |
| "grad_norm": 4.6715375481143795, |
| "learning_rate": 4.1666666666666667e-07, |
| "logits/chosen": -0.6943680047988892, |
| "logits/rejected": -0.7993655204772949, |
| "logps/chosen": -294.47052001953125, |
| "logps/rejected": -283.4633483886719, |
| "loss": 0.6883, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.04448702558875084, |
| "rewards/margins": 0.008819807320833206, |
| "rewards/rejected": 0.03566721826791763, |
| "step": 40 |
| }, |
| { |
| "dpo_lambda": 0.9958071112632751, |
| "epoch": 0.0858414027741429, |
| "grad_norm": 3.8258723543140745, |
| "learning_rate": 4.270833333333333e-07, |
| "logits/chosen": -0.7413933277130127, |
| "logits/rejected": -0.8862082958221436, |
| "logps/chosen": -313.7814025878906, |
| "logps/rejected": -279.2095642089844, |
| "loss": 0.6877, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.04902862012386322, |
| "rewards/margins": 0.0171800684183836, |
| "rewards/rejected": 0.03184855356812477, |
| "step": 41 |
| }, |
| { |
| "dpo_lambda": 0.99570232629776, |
| "epoch": 0.08793509552473175, |
| "grad_norm": 4.072117115897332, |
| "learning_rate": 4.375e-07, |
| "logits/chosen": -0.8718756437301636, |
| "logits/rejected": -0.8905589580535889, |
| "logps/chosen": -289.25311279296875, |
| "logps/rejected": -283.2568664550781, |
| "loss": 0.687, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.04595724865794182, |
| "rewards/margins": 0.012137094512581825, |
| "rewards/rejected": 0.03382015600800514, |
| "step": 42 |
| }, |
| { |
| "dpo_lambda": 0.9955974221229553, |
| "epoch": 0.0900287882753206, |
| "grad_norm": 7.88653117810627, |
| "learning_rate": 4.479166666666667e-07, |
| "logits/chosen": -0.8274166584014893, |
| "logits/rejected": -0.9210019707679749, |
| "logps/chosen": -324.5426940917969, |
| "logps/rejected": -277.363037109375, |
| "loss": 0.6838, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": 0.058937303721904755, |
| "rewards/margins": 0.025202667340636253, |
| "rewards/rejected": 0.03373463824391365, |
| "step": 43 |
| }, |
| { |
| "dpo_lambda": 0.9954925775527954, |
| "epoch": 0.09212248102590945, |
| "grad_norm": 5.275783711822973, |
| "learning_rate": 4.5833333333333327e-07, |
| "logits/chosen": -0.7992769479751587, |
| "logits/rejected": -0.8615429997444153, |
| "logps/chosen": -292.977294921875, |
| "logps/rejected": -282.3927001953125, |
| "loss": 0.6842, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.05305924639105797, |
| "rewards/margins": 0.024420510977506638, |
| "rewards/rejected": 0.028638731688261032, |
| "step": 44 |
| }, |
| { |
| "dpo_lambda": 0.9953879117965698, |
| "epoch": 0.0942161737764983, |
| "grad_norm": 3.3397512285849142, |
| "learning_rate": 4.6874999999999996e-07, |
| "logits/chosen": -0.8438299894332886, |
| "logits/rejected": -0.8847188949584961, |
| "logps/chosen": -263.0268249511719, |
| "logps/rejected": -230.08824157714844, |
| "loss": 0.6867, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": 0.051549993455410004, |
| "rewards/margins": 0.01370037067681551, |
| "rewards/rejected": 0.03784961998462677, |
| "step": 45 |
| }, |
| { |
| "dpo_lambda": 0.9952830672264099, |
| "epoch": 0.09630986652708715, |
| "grad_norm": 4.375598831962758, |
| "learning_rate": 4.791666666666667e-07, |
| "logits/chosen": -0.8429378867149353, |
| "logits/rejected": -0.8720081448554993, |
| "logps/chosen": -290.9970703125, |
| "logps/rejected": -254.0476531982422, |
| "loss": 0.6796, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.05708230659365654, |
| "rewards/margins": 0.023602774366736412, |
| "rewards/rejected": 0.03347952663898468, |
| "step": 46 |
| }, |
| { |
| "dpo_lambda": 0.99517822265625, |
| "epoch": 0.098403559277676, |
| "grad_norm": 7.220768015853875, |
| "learning_rate": 4.895833333333333e-07, |
| "logits/chosen": -0.8145819902420044, |
| "logits/rejected": -0.9115846753120422, |
| "logps/chosen": -321.8493347167969, |
| "logps/rejected": -276.99609375, |
| "loss": 0.6808, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.07153000682592392, |
| "rewards/margins": 0.023595348000526428, |
| "rewards/rejected": 0.04793466255068779, |
| "step": 47 |
| }, |
| { |
| "dpo_lambda": 0.9950733780860901, |
| "epoch": 0.10049725202826486, |
| "grad_norm": 4.4604551692806655, |
| "learning_rate": 5e-07, |
| "logits/chosen": -0.7160397171974182, |
| "logits/rejected": -0.8093741536140442, |
| "logps/chosen": -279.6042175292969, |
| "logps/rejected": -284.2084655761719, |
| "loss": 0.678, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": 0.058196570724248886, |
| "rewards/margins": 0.029132841154932976, |
| "rewards/rejected": 0.029063725844025612, |
| "step": 48 |
| }, |
| { |
| "dpo_lambda": 0.9949684739112854, |
| "epoch": 0.10259094477885371, |
| "grad_norm": 4.361132421326402, |
| "learning_rate": 4.999932966293553e-07, |
| "logits/chosen": -0.7594467401504517, |
| "logits/rejected": -0.8968175649642944, |
| "logps/chosen": -292.1831970214844, |
| "logps/rejected": -258.41021728515625, |
| "loss": 0.6795, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": 0.07236772030591965, |
| "rewards/margins": 0.03399732708930969, |
| "rewards/rejected": 0.03837038576602936, |
| "step": 49 |
| }, |
| { |
| "dpo_lambda": 0.9948636293411255, |
| "epoch": 0.10468463752944256, |
| "grad_norm": 3.9072947893099683, |
| "learning_rate": 4.999731868769026e-07, |
| "logits/chosen": -0.9222686886787415, |
| "logits/rejected": -0.9353721737861633, |
| "logps/chosen": -304.7239990234375, |
| "logps/rejected": -269.6137390136719, |
| "loss": 0.6826, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.0706743523478508, |
| "rewards/margins": 0.01939474046230316, |
| "rewards/rejected": 0.05127961188554764, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.10468463752944256, |
| "eval_dpo_lambda": 0.9947588443756104, |
| "eval_logits/chosen": -0.8412269949913025, |
| "eval_logits/rejected": -0.9094433188438416, |
| "eval_logps/chosen": -293.9556884765625, |
| "eval_logps/rejected": -267.0430908203125, |
| "eval_loss": 0.6803367733955383, |
| "eval_rewards/accuracies": 0.6690000295639038, |
| "eval_rewards/chosen": 0.06691199541091919, |
| "eval_rewards/margins": 0.027035199105739594, |
| "eval_rewards/rejected": 0.0398767925798893, |
| "eval_runtime": 566.7091, |
| "eval_samples_per_second": 3.529, |
| "eval_steps_per_second": 0.882, |
| "step": 50 |
| }, |
| { |
| "dpo_lambda": 0.9947589635848999, |
| "epoch": 0.1067783302800314, |
| "grad_norm": 3.4690982750836916, |
| "learning_rate": 4.99939671821067e-07, |
| "logits/chosen": -0.8632264137268066, |
| "logits/rejected": -0.9709175229072571, |
| "logps/chosen": -291.12359619140625, |
| "logps/rejected": -259.31390380859375, |
| "loss": 0.6863, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": 0.06085498631000519, |
| "rewards/margins": 0.01772325485944748, |
| "rewards/rejected": 0.043131738901138306, |
| "step": 51 |
| }, |
| { |
| "dpo_lambda": 0.99465411901474, |
| "epoch": 0.10887202303062025, |
| "grad_norm": 3.933263008508187, |
| "learning_rate": 4.998927532591591e-07, |
| "logits/chosen": -0.7605027556419373, |
| "logits/rejected": -0.8193937540054321, |
| "logps/chosen": -303.9796142578125, |
| "logps/rejected": -279.7144470214844, |
| "loss": 0.6833, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": 0.06420192122459412, |
| "rewards/margins": 0.015458992682397366, |
| "rewards/rejected": 0.048742931336164474, |
| "step": 52 |
| }, |
| { |
| "dpo_lambda": 0.9945492744445801, |
| "epoch": 0.1109657157812091, |
| "grad_norm": 4.1504325386412395, |
| "learning_rate": 4.998324337072792e-07, |
| "logits/chosen": -0.8832409381866455, |
| "logits/rejected": -0.9020228385925293, |
| "logps/chosen": -230.05926513671875, |
| "logps/rejected": -230.46888732910156, |
| "loss": 0.6786, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.06142224371433258, |
| "rewards/margins": 0.030234448611736298, |
| "rewards/rejected": 0.03118780255317688, |
| "step": 53 |
| }, |
| { |
| "dpo_lambda": 0.9944444298744202, |
| "epoch": 0.11305940853179795, |
| "grad_norm": 4.022336476891531, |
| "learning_rate": 4.997587164001815e-07, |
| "logits/chosen": -0.9572932720184326, |
| "logits/rejected": -0.8822497129440308, |
| "logps/chosen": -312.7522277832031, |
| "logps/rejected": -292.34979248046875, |
| "loss": 0.6729, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.0748954638838768, |
| "rewards/margins": 0.05051225423812866, |
| "rewards/rejected": 0.02438320592045784, |
| "step": 54 |
| }, |
| { |
| "dpo_lambda": 0.9943395853042603, |
| "epoch": 0.11515310128238682, |
| "grad_norm": 4.085909737303222, |
| "learning_rate": 4.996716052911017e-07, |
| "logits/chosen": -0.798669159412384, |
| "logits/rejected": -0.8256829380989075, |
| "logps/chosen": -270.40679931640625, |
| "logps/rejected": -253.59112548828125, |
| "loss": 0.677, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": 0.07010850310325623, |
| "rewards/margins": 0.03986787796020508, |
| "rewards/rejected": 0.030240625143051147, |
| "step": 55 |
| }, |
| { |
| "dpo_lambda": 0.9942349195480347, |
| "epoch": 0.11724679403297567, |
| "grad_norm": 4.021809912258076, |
| "learning_rate": 4.99571105051544e-07, |
| "logits/chosen": -0.8572608232498169, |
| "logits/rejected": -0.9068230390548706, |
| "logps/chosen": -269.0156555175781, |
| "logps/rejected": -263.5270690917969, |
| "loss": 0.6726, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": 0.0742337703704834, |
| "rewards/margins": 0.03828868642449379, |
| "rewards/rejected": 0.03594507277011871, |
| "step": 56 |
| }, |
| { |
| "dpo_lambda": 0.99413001537323, |
| "epoch": 0.11934048678356451, |
| "grad_norm": 4.4423528608842995, |
| "learning_rate": 4.994572210710314e-07, |
| "logits/chosen": -0.8098608255386353, |
| "logits/rejected": -0.9324737787246704, |
| "logps/chosen": -323.1205749511719, |
| "logps/rejected": -322.8743896484375, |
| "loss": 0.6707, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": 0.07381981611251831, |
| "rewards/margins": 0.053245287388563156, |
| "rewards/rejected": 0.020574528723955154, |
| "step": 57 |
| }, |
| { |
| "dpo_lambda": 0.9940251708030701, |
| "epoch": 0.12143417953415336, |
| "grad_norm": 6.2104322840923984, |
| "learning_rate": 4.993299594568162e-07, |
| "logits/chosen": -0.7725510001182556, |
| "logits/rejected": -0.882312536239624, |
| "logps/chosen": -274.6170654296875, |
| "logps/rejected": -282.4775390625, |
| "loss": 0.669, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.0651557445526123, |
| "rewards/margins": 0.04476066306233406, |
| "rewards/rejected": 0.020395075902342796, |
| "step": 58 |
| }, |
| { |
| "dpo_lambda": 0.9939203262329102, |
| "epoch": 0.12352787228474221, |
| "grad_norm": 4.039407324119509, |
| "learning_rate": 4.991893270335525e-07, |
| "logits/chosen": -0.8754050731658936, |
| "logits/rejected": -0.9039373397827148, |
| "logps/chosen": -300.01531982421875, |
| "logps/rejected": -246.3870086669922, |
| "loss": 0.6774, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": 0.0613943375647068, |
| "rewards/margins": 0.04010332375764847, |
| "rewards/rejected": 0.021291015669703484, |
| "step": 59 |
| }, |
| { |
| "dpo_lambda": 0.9938154816627502, |
| "epoch": 0.12562156503533106, |
| "grad_norm": 4.220351838843851, |
| "learning_rate": 4.990353313429303e-07, |
| "logits/chosen": -0.8983861804008484, |
| "logits/rejected": -0.9741111397743225, |
| "logps/chosen": -293.3563537597656, |
| "logps/rejected": -272.404052734375, |
| "loss": 0.6698, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.050274576991796494, |
| "rewards/margins": 0.04937704652547836, |
| "rewards/rejected": 0.0008975326200015843, |
| "step": 60 |
| }, |
| { |
| "dpo_lambda": 0.9937106370925903, |
| "epoch": 0.1277152577859199, |
| "grad_norm": 4.830185238287808, |
| "learning_rate": 4.988679806432711e-07, |
| "logits/chosen": -0.8061741590499878, |
| "logits/rejected": -0.8219490051269531, |
| "logps/chosen": -333.5423278808594, |
| "logps/rejected": -304.4486083984375, |
| "loss": 0.6691, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": 0.04406392201781273, |
| "rewards/margins": 0.0422632060945034, |
| "rewards/rejected": 0.0018007168546319008, |
| "step": 61 |
| }, |
| { |
| "dpo_lambda": 0.9936057329177856, |
| "epoch": 0.12980895053650876, |
| "grad_norm": 13.0411543579937, |
| "learning_rate": 4.986872839090852e-07, |
| "logits/chosen": -0.9093612432479858, |
| "logits/rejected": -0.987296998500824, |
| "logps/chosen": -323.12005615234375, |
| "logps/rejected": -277.0774841308594, |
| "loss": 0.6655, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.05420689284801483, |
| "rewards/margins": 0.06399476528167725, |
| "rewards/rejected": -0.009787879884243011, |
| "step": 62 |
| }, |
| { |
| "dpo_lambda": 0.9935011267662048, |
| "epoch": 0.1319026432870976, |
| "grad_norm": 7.307100303338513, |
| "learning_rate": 4.9849325083059e-07, |
| "logits/chosen": -0.9184948205947876, |
| "logits/rejected": -0.9709855914115906, |
| "logps/chosen": -288.5375061035156, |
| "logps/rejected": -257.5970458984375, |
| "loss": 0.6596, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.03467197343707085, |
| "rewards/margins": 0.061485495418310165, |
| "rewards/rejected": -0.02681351825594902, |
| "step": 63 |
| }, |
| { |
| "dpo_lambda": 0.9933962225914001, |
| "epoch": 0.13399633603768646, |
| "grad_norm": 3.6883053376031145, |
| "learning_rate": 4.982858918131906e-07, |
| "logits/chosen": -0.8752248287200928, |
| "logits/rejected": -0.918498158454895, |
| "logps/chosen": -278.55010986328125, |
| "logps/rejected": -274.21343994140625, |
| "loss": 0.675, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": 0.004433615133166313, |
| "rewards/margins": 0.035762809216976166, |
| "rewards/rejected": -0.031329195946455, |
| "step": 64 |
| }, |
| { |
| "dpo_lambda": 0.9932913780212402, |
| "epoch": 0.1360900287882753, |
| "grad_norm": 8.921064153450724, |
| "learning_rate": 4.980652179769217e-07, |
| "logits/chosen": -0.8184975385665894, |
| "logits/rejected": -0.8712335824966431, |
| "logps/chosen": -291.94488525390625, |
| "logps/rejected": -271.47967529296875, |
| "loss": 0.6645, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.01280883327126503, |
| "rewards/margins": 0.057765569537878036, |
| "rewards/rejected": -0.04495673626661301, |
| "step": 65 |
| }, |
| { |
| "dpo_lambda": 0.9931865334510803, |
| "epoch": 0.13818372153886418, |
| "grad_norm": 6.714346880863985, |
| "learning_rate": 4.978312411558517e-07, |
| "logits/chosen": -0.8184133172035217, |
| "logits/rejected": -0.8564844727516174, |
| "logps/chosen": -355.27081298828125, |
| "logps/rejected": -319.2922058105469, |
| "loss": 0.6674, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": 0.02395615540444851, |
| "rewards/margins": 0.07637568563222885, |
| "rewards/rejected": -0.05241953209042549, |
| "step": 66 |
| }, |
| { |
| "dpo_lambda": 0.9930816888809204, |
| "epoch": 0.14027741428945303, |
| "grad_norm": 10.679575076032895, |
| "learning_rate": 4.975839738974473e-07, |
| "logits/chosen": -0.7999946475028992, |
| "logits/rejected": -0.8521921634674072, |
| "logps/chosen": -330.9016418457031, |
| "logps/rejected": -294.4024963378906, |
| "loss": 0.6446, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.010321049951016903, |
| "rewards/margins": 0.07487591356039047, |
| "rewards/rejected": -0.0645548552274704, |
| "step": 67 |
| }, |
| { |
| "dpo_lambda": 0.9929770231246948, |
| "epoch": 0.14237110704004188, |
| "grad_norm": 7.875861769063148, |
| "learning_rate": 4.97323429461901e-07, |
| "logits/chosen": -0.8829125165939331, |
| "logits/rejected": -0.9044405817985535, |
| "logps/chosen": -282.6078796386719, |
| "logps/rejected": -283.01708984375, |
| "loss": 0.6603, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.018198691308498383, |
| "rewards/margins": 0.04956141486763954, |
| "rewards/rejected": -0.06776010245084763, |
| "step": 68 |
| }, |
| { |
| "dpo_lambda": 0.9928721785545349, |
| "epoch": 0.14446479979063073, |
| "grad_norm": 4.618590469666639, |
| "learning_rate": 4.970496218214204e-07, |
| "logits/chosen": -0.8894454836845398, |
| "logits/rejected": -0.8717415928840637, |
| "logps/chosen": -294.20208740234375, |
| "logps/rejected": -314.8604431152344, |
| "loss": 0.6688, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.017065059393644333, |
| "rewards/margins": 0.0473010316491127, |
| "rewards/rejected": -0.06436608731746674, |
| "step": 69 |
| }, |
| { |
| "dpo_lambda": 0.9927672743797302, |
| "epoch": 0.14655849254121958, |
| "grad_norm": 6.326104311922183, |
| "learning_rate": 4.967625656594781e-07, |
| "logits/chosen": -0.8711340427398682, |
| "logits/rejected": -0.9299571514129639, |
| "logps/chosen": -270.4837646484375, |
| "logps/rejected": -288.94024658203125, |
| "loss": 0.666, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.007623748853802681, |
| "rewards/margins": 0.05371168255805969, |
| "rewards/rejected": -0.061335429549217224, |
| "step": 70 |
| }, |
| { |
| "dpo_lambda": 0.9926624894142151, |
| "epoch": 0.14865218529180843, |
| "grad_norm": 5.426572331511348, |
| "learning_rate": 4.964622763700252e-07, |
| "logits/chosen": -0.9065719842910767, |
| "logits/rejected": -0.937716007232666, |
| "logps/chosen": -359.8345947265625, |
| "logps/rejected": -333.8294372558594, |
| "loss": 0.6646, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.006900262087583542, |
| "rewards/margins": 0.06472197920084, |
| "rewards/rejected": -0.07162223756313324, |
| "step": 71 |
| }, |
| { |
| "dpo_lambda": 0.9925575852394104, |
| "epoch": 0.15074587804239728, |
| "grad_norm": 5.887744964119917, |
| "learning_rate": 4.961487700566646e-07, |
| "logits/chosen": -0.8798828721046448, |
| "logits/rejected": -0.9241186380386353, |
| "logps/chosen": -349.4935607910156, |
| "logps/rejected": -300.86651611328125, |
| "loss": 0.6537, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": 0.019017428159713745, |
| "rewards/margins": 0.08565981686115265, |
| "rewards/rejected": -0.0666423887014389, |
| "step": 72 |
| }, |
| { |
| "dpo_lambda": 0.9924527406692505, |
| "epoch": 0.15283957079298613, |
| "grad_norm": 5.112043217977206, |
| "learning_rate": 4.958220635317885e-07, |
| "logits/chosen": -0.874078094959259, |
| "logits/rejected": -0.9611667990684509, |
| "logps/chosen": -358.426513671875, |
| "logps/rejected": -303.3841552734375, |
| "loss": 0.6627, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.017709314823150635, |
| "rewards/margins": 0.08091504871845245, |
| "rewards/rejected": -0.09862436354160309, |
| "step": 73 |
| }, |
| { |
| "dpo_lambda": 0.9923480749130249, |
| "epoch": 0.15493326354357498, |
| "grad_norm": 4.166549228246831, |
| "learning_rate": 4.954821743156767e-07, |
| "logits/chosen": -0.9340280890464783, |
| "logits/rejected": -0.8813725113868713, |
| "logps/chosen": -250.7559814453125, |
| "logps/rejected": -250.04638671875, |
| "loss": 0.6608, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.008466021157801151, |
| "rewards/margins": 0.06686487793922424, |
| "rewards/rejected": -0.07533089816570282, |
| "step": 74 |
| }, |
| { |
| "dpo_lambda": 0.992243230342865, |
| "epoch": 0.15702695629416383, |
| "grad_norm": 4.579667286901192, |
| "learning_rate": 4.951291206355559e-07, |
| "logits/chosen": -0.8417796492576599, |
| "logits/rejected": -0.8593603372573853, |
| "logps/chosen": -278.1746826171875, |
| "logps/rejected": -288.68060302734375, |
| "loss": 0.6551, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.008493858389556408, |
| "rewards/margins": 0.0969361662864685, |
| "rewards/rejected": -0.10543002933263779, |
| "step": 75 |
| }, |
| { |
| "dpo_lambda": 0.9921383857727051, |
| "epoch": 0.15912064904475268, |
| "grad_norm": 4.441173202852156, |
| "learning_rate": 4.947629214246236e-07, |
| "logits/chosen": -0.824394166469574, |
| "logits/rejected": -0.9439455270767212, |
| "logps/chosen": -264.31573486328125, |
| "logps/rejected": -238.005615234375, |
| "loss": 0.6518, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.01480570062994957, |
| "rewards/margins": 0.118388332426548, |
| "rewards/rejected": -0.10358262807130814, |
| "step": 76 |
| }, |
| { |
| "dpo_lambda": 0.9920335412025452, |
| "epoch": 0.16121434179534153, |
| "grad_norm": 4.800895103736595, |
| "learning_rate": 4.943835963210323e-07, |
| "logits/chosen": -0.9381400942802429, |
| "logits/rejected": -0.9885379672050476, |
| "logps/chosen": -291.7290344238281, |
| "logps/rejected": -253.0125274658203, |
| "loss": 0.6549, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.015666170045733452, |
| "rewards/margins": 0.09368825703859329, |
| "rewards/rejected": -0.1093544289469719, |
| "step": 77 |
| }, |
| { |
| "dpo_lambda": 0.9919286370277405, |
| "epoch": 0.16330803454593038, |
| "grad_norm": 6.9168703022954725, |
| "learning_rate": 4.939911656668361e-07, |
| "logits/chosen": -0.8449192047119141, |
| "logits/rejected": -0.9392167329788208, |
| "logps/chosen": -314.6385803222656, |
| "logps/rejected": -261.46875, |
| "loss": 0.6502, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.029134994372725487, |
| "rewards/margins": 0.11234299838542938, |
| "rewards/rejected": -0.08320800215005875, |
| "step": 78 |
| }, |
| { |
| "dpo_lambda": 0.9918240308761597, |
| "epoch": 0.16540172729651922, |
| "grad_norm": 7.545177066125938, |
| "learning_rate": 4.935856505068998e-07, |
| "logits/chosen": -0.9078419804573059, |
| "logits/rejected": -1.0092755556106567, |
| "logps/chosen": -275.2136535644531, |
| "logps/rejected": -308.03533935546875, |
| "loss": 0.6369, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": 0.024624880403280258, |
| "rewards/margins": 0.13832628726959229, |
| "rewards/rejected": -0.11370141804218292, |
| "step": 79 |
| }, |
| { |
| "dpo_lambda": 0.991719126701355, |
| "epoch": 0.16749542004710807, |
| "grad_norm": 4.95330943717671, |
| "learning_rate": 4.93167072587771e-07, |
| "logits/chosen": -0.8874943256378174, |
| "logits/rejected": -0.9544984102249146, |
| "logps/chosen": -335.6016540527344, |
| "logps/rejected": -310.4072570800781, |
| "loss": 0.6488, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.06745388358831406, |
| "rewards/margins": 0.10092146694660187, |
| "rewards/rejected": -0.16837534308433533, |
| "step": 80 |
| }, |
| { |
| "dpo_lambda": 0.9916142821311951, |
| "epoch": 0.16958911279769695, |
| "grad_norm": 5.518201533595615, |
| "learning_rate": 4.92735454356513e-07, |
| "logits/chosen": -0.9439139366149902, |
| "logits/rejected": -0.9663654565811157, |
| "logps/chosen": -269.6962585449219, |
| "logps/rejected": -272.7530212402344, |
| "loss": 0.6392, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.0002646269276738167, |
| "rewards/margins": 0.1335701048374176, |
| "rewards/rejected": -0.13383471965789795, |
| "step": 81 |
| }, |
| { |
| "dpo_lambda": 0.9915094375610352, |
| "epoch": 0.1716828055482858, |
| "grad_norm": 6.030336679079051, |
| "learning_rate": 4.922908189595017e-07, |
| "logits/chosen": -0.9288738369941711, |
| "logits/rejected": -0.9926230311393738, |
| "logps/chosen": -279.9482421875, |
| "logps/rejected": -245.69894409179688, |
| "loss": 0.6471, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.009446481242775917, |
| "rewards/margins": 0.1037771925330162, |
| "rewards/rejected": -0.09433071315288544, |
| "step": 82 |
| }, |
| { |
| "dpo_lambda": 0.9914045929908752, |
| "epoch": 0.17377649829887465, |
| "grad_norm": 6.812109507351604, |
| "learning_rate": 4.918331902411841e-07, |
| "logits/chosen": -0.8729565143585205, |
| "logits/rejected": -0.9138444662094116, |
| "logps/chosen": -292.5542907714844, |
| "logps/rejected": -281.9892272949219, |
| "loss": 0.6523, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.008294099941849709, |
| "rewards/margins": 0.10660440474748611, |
| "rewards/rejected": -0.11489850282669067, |
| "step": 83 |
| }, |
| { |
| "dpo_lambda": 0.9912997484207153, |
| "epoch": 0.1758701910494635, |
| "grad_norm": 7.516475417347365, |
| "learning_rate": 4.913625927427995e-07, |
| "logits/chosen": -0.9265223741531372, |
| "logits/rejected": -0.9852555394172668, |
| "logps/chosen": -332.3023681640625, |
| "logps/rejected": -303.4096984863281, |
| "loss": 0.625, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.007811751216650009, |
| "rewards/margins": 0.1576412171125412, |
| "rewards/rejected": -0.1654529720544815, |
| "step": 84 |
| }, |
| { |
| "dpo_lambda": 0.9911948442459106, |
| "epoch": 0.17796388380005235, |
| "grad_norm": 5.373997547948311, |
| "learning_rate": 4.908790517010636e-07, |
| "logits/chosen": -0.9465526342391968, |
| "logits/rejected": -0.9650004506111145, |
| "logps/chosen": -305.8551330566406, |
| "logps/rejected": -287.52679443359375, |
| "loss": 0.6332, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.002578308340162039, |
| "rewards/margins": 0.12680523097515106, |
| "rewards/rejected": -0.12938354909420013, |
| "step": 85 |
| }, |
| { |
| "dpo_lambda": 0.9910901784896851, |
| "epoch": 0.1800575765506412, |
| "grad_norm": 5.931519307525962, |
| "learning_rate": 4.903825930468148e-07, |
| "logits/chosen": -0.8247543573379517, |
| "logits/rejected": -0.9537917375564575, |
| "logps/chosen": -288.71148681640625, |
| "logps/rejected": -268.2547302246094, |
| "loss": 0.623, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.024831930175423622, |
| "rewards/margins": 0.19303402304649353, |
| "rewards/rejected": -0.2178659439086914, |
| "step": 86 |
| }, |
| { |
| "dpo_lambda": 0.9909853339195251, |
| "epoch": 0.18215126930123005, |
| "grad_norm": 6.129440307717882, |
| "learning_rate": 4.898732434036243e-07, |
| "logits/chosen": -0.8919886350631714, |
| "logits/rejected": -0.9316573143005371, |
| "logps/chosen": -299.3219909667969, |
| "logps/rejected": -325.168212890625, |
| "loss": 0.6235, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.023229926824569702, |
| "rewards/margins": 0.1745581179857254, |
| "rewards/rejected": -0.1977880299091339, |
| "step": 87 |
| }, |
| { |
| "dpo_lambda": 0.9908804893493652, |
| "epoch": 0.1842449620518189, |
| "grad_norm": 21.191472422356895, |
| "learning_rate": 4.893510300863676e-07, |
| "logits/chosen": -0.8724699020385742, |
| "logits/rejected": -0.894172191619873, |
| "logps/chosen": -249.15235900878906, |
| "logps/rejected": -276.5699768066406, |
| "loss": 0.6509, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.10194307565689087, |
| "rewards/margins": 0.10438381880521774, |
| "rewards/rejected": -0.2063269019126892, |
| "step": 88 |
| }, |
| { |
| "dpo_lambda": 0.9907756447792053, |
| "epoch": 0.18633865480240774, |
| "grad_norm": 9.630094496567592, |
| "learning_rate": 4.8881598109976e-07, |
| "logits/chosen": -0.8519224524497986, |
| "logits/rejected": -0.9527184367179871, |
| "logps/chosen": -374.3948974609375, |
| "logps/rejected": -337.03668212890625, |
| "loss": 0.6426, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.08539784699678421, |
| "rewards/margins": 0.12617787718772888, |
| "rewards/rejected": -0.2115756869316101, |
| "step": 89 |
| }, |
| { |
| "dpo_lambda": 0.9906708002090454, |
| "epoch": 0.1884323475529966, |
| "grad_norm": 6.3578631755387525, |
| "learning_rate": 4.882681251368548e-07, |
| "logits/chosen": -0.8806661367416382, |
| "logits/rejected": -0.8995652794837952, |
| "logps/chosen": -294.17474365234375, |
| "logps/rejected": -294.7281494140625, |
| "loss": 0.6287, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.12962456047534943, |
| "rewards/margins": 0.16316621005535126, |
| "rewards/rejected": -0.29279080033302307, |
| "step": 90 |
| }, |
| { |
| "dpo_lambda": 0.9905661344528198, |
| "epoch": 0.19052604030358544, |
| "grad_norm": 6.600987454158478, |
| "learning_rate": 4.877074915775048e-07, |
| "logits/chosen": -0.8745786547660828, |
| "logits/rejected": -0.8542971611022949, |
| "logps/chosen": -327.0505065917969, |
| "logps/rejected": -289.8307800292969, |
| "loss": 0.6266, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.08782470226287842, |
| "rewards/margins": 0.18679843842983246, |
| "rewards/rejected": -0.2746231257915497, |
| "step": 91 |
| }, |
| { |
| "dpo_lambda": 0.9904612898826599, |
| "epoch": 0.1926197330541743, |
| "grad_norm": 8.081977238915549, |
| "learning_rate": 4.871341104867864e-07, |
| "logits/chosen": -0.8323448896408081, |
| "logits/rejected": -0.9532727599143982, |
| "logps/chosen": -301.7206726074219, |
| "logps/rejected": -277.83123779296875, |
| "loss": 0.6013, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.041278161108493805, |
| "rewards/margins": 0.25224485993385315, |
| "rewards/rejected": -0.29352301359176636, |
| "step": 92 |
| }, |
| { |
| "dpo_lambda": 0.9903563857078552, |
| "epoch": 0.19471342580476314, |
| "grad_norm": 11.457731875661583, |
| "learning_rate": 4.865480126133871e-07, |
| "logits/chosen": -0.8399688005447388, |
| "logits/rejected": -0.9014586210250854, |
| "logps/chosen": -277.01605224609375, |
| "logps/rejected": -324.4666442871094, |
| "loss": 0.618, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.08516222983598709, |
| "rewards/margins": 0.19609113037586212, |
| "rewards/rejected": -0.2812533378601074, |
| "step": 93 |
| }, |
| { |
| "dpo_lambda": 0.9902515411376953, |
| "epoch": 0.196807118555352, |
| "grad_norm": 8.795130599904098, |
| "learning_rate": 4.859492293879573e-07, |
| "logits/chosen": -0.8535427451133728, |
| "logits/rejected": -0.8940081000328064, |
| "logps/chosen": -243.99441528320312, |
| "logps/rejected": -254.5758819580078, |
| "loss": 0.6554, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.05271986871957779, |
| "rewards/margins": 0.1381668746471405, |
| "rewards/rejected": -0.1908867508172989, |
| "step": 94 |
| }, |
| { |
| "dpo_lambda": 0.9901466965675354, |
| "epoch": 0.19890081130594087, |
| "grad_norm": 22.46452853252336, |
| "learning_rate": 4.853377929214243e-07, |
| "logits/chosen": -0.8657770156860352, |
| "logits/rejected": -0.8967228531837463, |
| "logps/chosen": -309.3226623535156, |
| "logps/rejected": -328.7403564453125, |
| "loss": 0.6138, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.12215033918619156, |
| "rewards/margins": 0.18180522322654724, |
| "rewards/rejected": -0.3039555251598358, |
| "step": 95 |
| }, |
| { |
| "dpo_lambda": 0.9900418519973755, |
| "epoch": 0.20099450405652972, |
| "grad_norm": 6.7949580684730515, |
| "learning_rate": 4.847137360032699e-07, |
| "logits/chosen": -0.9191737771034241, |
| "logits/rejected": -0.9387893676757812, |
| "logps/chosen": -324.2237854003906, |
| "logps/rejected": -314.5294189453125, |
| "loss": 0.6324, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.09853880852460861, |
| "rewards/margins": 0.2057497352361679, |
| "rewards/rejected": -0.3042885363101959, |
| "step": 96 |
| }, |
| { |
| "dpo_lambda": 0.9899371862411499, |
| "epoch": 0.20308819680711857, |
| "grad_norm": 6.158606287373961, |
| "learning_rate": 4.84077092099773e-07, |
| "logits/chosen": -0.8146138787269592, |
| "logits/rejected": -0.8111870884895325, |
| "logps/chosen": -346.63946533203125, |
| "logps/rejected": -317.3700866699219, |
| "loss": 0.6245, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.11687865853309631, |
| "rewards/margins": 0.15650136768817902, |
| "rewards/rejected": -0.2733800411224365, |
| "step": 97 |
| }, |
| { |
| "dpo_lambda": 0.98983234167099, |
| "epoch": 0.20518188955770741, |
| "grad_norm": 5.273144558536801, |
| "learning_rate": 4.834278953522137e-07, |
| "logits/chosen": -0.8933238983154297, |
| "logits/rejected": -0.9000183343887329, |
| "logps/chosen": -341.7193298339844, |
| "logps/rejected": -319.6878662109375, |
| "loss": 0.6252, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.05204794183373451, |
| "rewards/margins": 0.19676589965820312, |
| "rewards/rejected": -0.24881383776664734, |
| "step": 98 |
| }, |
| { |
| "dpo_lambda": 0.9897274374961853, |
| "epoch": 0.20727558230829626, |
| "grad_norm": 13.514515103851846, |
| "learning_rate": 4.827661805750437e-07, |
| "logits/chosen": -0.8712120652198792, |
| "logits/rejected": -0.8879258632659912, |
| "logps/chosen": -251.89381408691406, |
| "logps/rejected": -281.3234558105469, |
| "loss": 0.6548, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.11316128820180893, |
| "rewards/margins": 0.16183218359947205, |
| "rewards/rejected": -0.2749934792518616, |
| "step": 99 |
| }, |
| { |
| "dpo_lambda": 0.9896226525306702, |
| "epoch": 0.2093692750588851, |
| "grad_norm": 10.260926722198429, |
| "learning_rate": 4.820919832540181e-07, |
| "logits/chosen": -0.8627618551254272, |
| "logits/rejected": -0.9980306029319763, |
| "logps/chosen": -315.52777099609375, |
| "logps/rejected": -257.67620849609375, |
| "loss": 0.5951, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.033613838255405426, |
| "rewards/margins": 0.22266733646392822, |
| "rewards/rejected": -0.25628116726875305, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.2093692750588851, |
| "eval_dpo_lambda": 0.9895178079605103, |
| "eval_logits/chosen": -0.8666641712188721, |
| "eval_logits/rejected": -0.919507622718811, |
| "eval_logps/chosen": -309.2590637207031, |
| "eval_logps/rejected": -298.4849853515625, |
| "eval_loss": 0.6223126649856567, |
| "eval_rewards/accuracies": 0.7129999995231628, |
| "eval_rewards/chosen": -0.08612177520990372, |
| "eval_rewards/margins": 0.18842005729675293, |
| "eval_rewards/rejected": -0.27454182505607605, |
| "eval_runtime": 560.924, |
| "eval_samples_per_second": 3.566, |
| "eval_steps_per_second": 0.891, |
| "step": 100 |
| }, |
| { |
| "dpo_lambda": 0.9895177483558655, |
| "epoch": 0.21146296780947396, |
| "grad_norm": 11.934315941165567, |
| "learning_rate": 4.814053395442932e-07, |
| "logits/chosen": -0.9160792231559753, |
| "logits/rejected": -0.958967924118042, |
| "logps/chosen": -295.62933349609375, |
| "logps/rejected": -283.5747985839844, |
| "loss": 0.6385, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.10350628197193146, |
| "rewards/margins": 0.18040822446346283, |
| "rewards/rejected": -0.2839145362377167, |
| "step": 101 |
| }, |
| { |
| "dpo_lambda": 0.9894131422042847, |
| "epoch": 0.2135566605600628, |
| "grad_norm": 13.96418452441393, |
| "learning_rate": 4.807062862684873e-07, |
| "logits/chosen": -0.8515546917915344, |
| "logits/rejected": -0.8713206648826599, |
| "logps/chosen": -274.0570068359375, |
| "logps/rejected": -318.78070068359375, |
| "loss": 0.6274, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.14027418196201324, |
| "rewards/margins": 0.13299782574176788, |
| "rewards/rejected": -0.27327200770378113, |
| "step": 102 |
| }, |
| { |
| "dpo_lambda": 0.98930823802948, |
| "epoch": 0.21565035331065166, |
| "grad_norm": 8.696396287402754, |
| "learning_rate": 4.799948609147061e-07, |
| "logits/chosen": -0.8648073673248291, |
| "logits/rejected": -0.9215875267982483, |
| "logps/chosen": -344.8481140136719, |
| "logps/rejected": -380.5777893066406, |
| "loss": 0.6581, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.17415529489517212, |
| "rewards/margins": 0.04592770338058472, |
| "rewards/rejected": -0.22008298337459564, |
| "step": 103 |
| }, |
| { |
| "dpo_lambda": 0.9892033934593201, |
| "epoch": 0.2177440460612405, |
| "grad_norm": 13.71055022495119, |
| "learning_rate": 4.792711016345321e-07, |
| "logits/chosen": -0.8732472658157349, |
| "logits/rejected": -0.9386597275733948, |
| "logps/chosen": -254.61373901367188, |
| "logps/rejected": -265.4243469238281, |
| "loss": 0.6175, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": -0.06950341910123825, |
| "rewards/margins": 0.21822991967201233, |
| "rewards/rejected": -0.2877333462238312, |
| "step": 104 |
| }, |
| { |
| "dpo_lambda": 0.9890985488891602, |
| "epoch": 0.21983773881182936, |
| "grad_norm": 13.927329236949847, |
| "learning_rate": 4.785350472409791e-07, |
| "logits/chosen": -0.8163785338401794, |
| "logits/rejected": -0.862995982170105, |
| "logps/chosen": -336.977783203125, |
| "logps/rejected": -323.1407165527344, |
| "loss": 0.6072, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.0720912292599678, |
| "rewards/margins": 0.2766712009906769, |
| "rewards/rejected": -0.3487624526023865, |
| "step": 105 |
| }, |
| { |
| "dpo_lambda": 0.9889937043190002, |
| "epoch": 0.2219314315624182, |
| "grad_norm": 17.021289496777705, |
| "learning_rate": 4.777867372064105e-07, |
| "logits/chosen": -0.8436889052391052, |
| "logits/rejected": -0.8120547533035278, |
| "logps/chosen": -302.01715087890625, |
| "logps/rejected": -307.5523681640625, |
| "loss": 0.6113, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.051165711134672165, |
| "rewards/margins": 0.24847164750099182, |
| "rewards/rejected": -0.2996373772621155, |
| "step": 106 |
| }, |
| { |
| "dpo_lambda": 0.9888888001441956, |
| "epoch": 0.22402512431300706, |
| "grad_norm": 16.04000803772397, |
| "learning_rate": 4.770262116604223e-07, |
| "logits/chosen": -0.7302054166793823, |
| "logits/rejected": -0.7980797290802002, |
| "logps/chosen": -311.28961181640625, |
| "logps/rejected": -275.90948486328125, |
| "loss": 0.6348, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.109623983502388, |
| "rewards/margins": 0.18497368693351746, |
| "rewards/rejected": -0.29459768533706665, |
| "step": 107 |
| }, |
| { |
| "dpo_lambda": 0.9887839555740356, |
| "epoch": 0.2261188170635959, |
| "grad_norm": 15.39744839277282, |
| "learning_rate": 4.7625351138769166e-07, |
| "logits/chosen": -0.8831318020820618, |
| "logits/rejected": -0.9750317335128784, |
| "logps/chosen": -317.7901306152344, |
| "logps/rejected": -317.9429016113281, |
| "loss": 0.6099, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.16366863250732422, |
| "rewards/margins": 0.2613745927810669, |
| "rewards/rejected": -0.4250431954860687, |
| "step": 108 |
| }, |
| { |
| "dpo_lambda": 0.9886792898178101, |
| "epoch": 0.22821250981418476, |
| "grad_norm": 22.120796176992506, |
| "learning_rate": 4.75468677825789e-07, |
| "logits/chosen": -0.796947717666626, |
| "logits/rejected": -0.8122372031211853, |
| "logps/chosen": -347.1046447753906, |
| "logps/rejected": -348.04705810546875, |
| "loss": 0.6219, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.19824260473251343, |
| "rewards/margins": 0.1869257688522339, |
| "rewards/rejected": -0.3851684033870697, |
| "step": 109 |
| }, |
| { |
| "dpo_lambda": 0.9885744452476501, |
| "epoch": 0.23030620256477363, |
| "grad_norm": 22.21838839500956, |
| "learning_rate": 4.7467175306295647e-07, |
| "logits/chosen": -0.8764458894729614, |
| "logits/rejected": -0.8621913194656372, |
| "logps/chosen": -290.6605224609375, |
| "logps/rejected": -304.8687438964844, |
| "loss": 0.6084, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.20384612679481506, |
| "rewards/margins": 0.18661168217658997, |
| "rewards/rejected": -0.39045780897140503, |
| "step": 110 |
| }, |
| { |
| "dpo_lambda": 0.9884696006774902, |
| "epoch": 0.23239989531536248, |
| "grad_norm": 15.44321482685626, |
| "learning_rate": 4.7386277983585053e-07, |
| "logits/chosen": -0.8445209264755249, |
| "logits/rejected": -0.8973492980003357, |
| "logps/chosen": -310.8070983886719, |
| "logps/rejected": -336.9670104980469, |
| "loss": 0.6198, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.22197970747947693, |
| "rewards/margins": 0.25436967611312866, |
| "rewards/rejected": -0.4763493537902832, |
| "step": 111 |
| }, |
| { |
| "dpo_lambda": 0.9883647561073303, |
| "epoch": 0.23449358806595133, |
| "grad_norm": 21.734524282209797, |
| "learning_rate": 4.7304180152725024e-07, |
| "logits/chosen": -0.7842965126037598, |
| "logits/rejected": -0.7857354283332825, |
| "logps/chosen": -340.5823974609375, |
| "logps/rejected": -327.64093017578125, |
| "loss": 0.5882, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.17074204981327057, |
| "rewards/margins": 0.3331073522567749, |
| "rewards/rejected": -0.5038493871688843, |
| "step": 112 |
| }, |
| { |
| "dpo_lambda": 0.9882599115371704, |
| "epoch": 0.23658728081654018, |
| "grad_norm": 17.283697072731332, |
| "learning_rate": 4.7220886216373085e-07, |
| "logits/chosen": -0.834023654460907, |
| "logits/rejected": -0.8667508363723755, |
| "logps/chosen": -341.08538818359375, |
| "logps/rejected": -288.1911315917969, |
| "loss": 0.6443, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.27939462661743164, |
| "rewards/margins": 0.20295719802379608, |
| "rewards/rejected": -0.48235180974006653, |
| "step": 113 |
| }, |
| { |
| "dpo_lambda": 0.9881552457809448, |
| "epoch": 0.23868097356712903, |
| "grad_norm": 23.58869951254614, |
| "learning_rate": 4.7136400641330245e-07, |
| "logits/chosen": -0.8713721036911011, |
| "logits/rejected": -0.8854274153709412, |
| "logps/chosen": -334.2297058105469, |
| "logps/rejected": -340.6756591796875, |
| "loss": 0.5903, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.16744357347488403, |
| "rewards/margins": 0.3070833086967468, |
| "rewards/rejected": -0.47452688217163086, |
| "step": 114 |
| }, |
| { |
| "dpo_lambda": 0.9880504012107849, |
| "epoch": 0.24077466631771788, |
| "grad_norm": 15.006316309545257, |
| "learning_rate": 4.70507279583015e-07, |
| "logits/chosen": -0.7990316152572632, |
| "logits/rejected": -0.9022696018218994, |
| "logps/chosen": -310.5735168457031, |
| "logps/rejected": -295.2878112792969, |
| "loss": 0.6205, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.14761359989643097, |
| "rewards/margins": 0.22134283185005188, |
| "rewards/rejected": -0.36895644664764404, |
| "step": 115 |
| }, |
| { |
| "dpo_lambda": 0.9879454970359802, |
| "epoch": 0.24286835906830673, |
| "grad_norm": 11.83329907541787, |
| "learning_rate": 4.6963872761652834e-07, |
| "logits/chosen": -0.7804672718048096, |
| "logits/rejected": -0.8577914834022522, |
| "logps/chosen": -336.4219665527344, |
| "logps/rejected": -331.4212341308594, |
| "loss": 0.6112, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": -0.18096496164798737, |
| "rewards/margins": 0.20200692117214203, |
| "rewards/rejected": -0.3829718828201294, |
| "step": 116 |
| }, |
| { |
| "dpo_lambda": 0.9878406524658203, |
| "epoch": 0.24496205181889558, |
| "grad_norm": 23.71777196224923, |
| "learning_rate": 4.687583970916486e-07, |
| "logits/chosen": -0.8969647884368896, |
| "logits/rejected": -0.885977029800415, |
| "logps/chosen": -370.19537353515625, |
| "logps/rejected": -361.7889709472656, |
| "loss": 0.5848, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.09151718765497208, |
| "rewards/margins": 0.2100907266139984, |
| "rewards/rejected": -0.3016079068183899, |
| "step": 117 |
| }, |
| { |
| "dpo_lambda": 0.9877358078956604, |
| "epoch": 0.24705574456948443, |
| "grad_norm": 10.243313554179645, |
| "learning_rate": 4.6786633521783005e-07, |
| "logits/chosen": -0.8844261765480042, |
| "logits/rejected": -0.8958966732025146, |
| "logps/chosen": -304.04376220703125, |
| "logps/rejected": -314.54034423828125, |
| "loss": 0.6086, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.05434232950210571, |
| "rewards/margins": 0.1959638148546219, |
| "rewards/rejected": -0.2503061294555664, |
| "step": 118 |
| }, |
| { |
| "dpo_lambda": 0.9876309633255005, |
| "epoch": 0.24914943732007327, |
| "grad_norm": 14.635815803976675, |
| "learning_rate": 4.669625898336438e-07, |
| "logits/chosen": -0.8229978084564209, |
| "logits/rejected": -0.9754506945610046, |
| "logps/chosen": -313.97821044921875, |
| "logps/rejected": -286.31610107421875, |
| "loss": 0.6014, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.030857756733894348, |
| "rewards/margins": 0.2675139904022217, |
| "rewards/rejected": -0.2983717620372772, |
| "step": 119 |
| }, |
| { |
| "dpo_lambda": 0.9875260591506958, |
| "epoch": 0.2512431300706621, |
| "grad_norm": 14.480284643195642, |
| "learning_rate": 4.6604720940421207e-07, |
| "logits/chosen": -0.9286684989929199, |
| "logits/rejected": -0.9161196947097778, |
| "logps/chosen": -287.4082336425781, |
| "logps/rejected": -302.98394775390625, |
| "loss": 0.6246, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.07125397026538849, |
| "rewards/margins": 0.1770327091217041, |
| "rewards/rejected": -0.2482866793870926, |
| "step": 120 |
| }, |
| { |
| "dpo_lambda": 0.987421452999115, |
| "epoch": 0.253336822821251, |
| "grad_norm": 23.772174182910327, |
| "learning_rate": 4.651202430186092e-07, |
| "logits/chosen": -0.8641669750213623, |
| "logits/rejected": -0.9186245203018188, |
| "logps/chosen": -300.3369445800781, |
| "logps/rejected": -298.85186767578125, |
| "loss": 0.5863, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.03199971839785576, |
| "rewards/margins": 0.27820879220962524, |
| "rewards/rejected": -0.2462090402841568, |
| "step": 121 |
| }, |
| { |
| "dpo_lambda": 0.9873165488243103, |
| "epoch": 0.2554305155718398, |
| "grad_norm": 9.02656815319192, |
| "learning_rate": 4.6418174038722924e-07, |
| "logits/chosen": -0.7821521162986755, |
| "logits/rejected": -0.8236280679702759, |
| "logps/chosen": -308.3457946777344, |
| "logps/rejected": -312.6419372558594, |
| "loss": 0.6026, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": 0.005551555659621954, |
| "rewards/margins": 0.258663147687912, |
| "rewards/rejected": -0.2531115412712097, |
| "step": 122 |
| }, |
| { |
| "dpo_lambda": 0.9872117638587952, |
| "epoch": 0.25752420832242867, |
| "grad_norm": 12.797039121653263, |
| "learning_rate": 4.6323175183912023e-07, |
| "logits/chosen": -0.8209193348884583, |
| "logits/rejected": -0.8819643259048462, |
| "logps/chosen": -309.806640625, |
| "logps/rejected": -299.98272705078125, |
| "loss": 0.5909, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": 0.021201107650995255, |
| "rewards/margins": 0.2730109691619873, |
| "rewards/rejected": -0.25180989503860474, |
| "step": 123 |
| }, |
| { |
| "dpo_lambda": 0.9871068596839905, |
| "epoch": 0.2596179010730175, |
| "grad_norm": 7.647022983322272, |
| "learning_rate": 4.6227032831928483e-07, |
| "logits/chosen": -0.8404148817062378, |
| "logits/rejected": -0.9573003649711609, |
| "logps/chosen": -296.4405822753906, |
| "logps/rejected": -270.74261474609375, |
| "loss": 0.586, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.0421200767159462, |
| "rewards/margins": 0.28268861770629883, |
| "rewards/rejected": -0.24056854844093323, |
| "step": 124 |
| }, |
| { |
| "dpo_lambda": 0.9870020151138306, |
| "epoch": 0.26171159382360637, |
| "grad_norm": 11.869445654498206, |
| "learning_rate": 4.612975213859487e-07, |
| "logits/chosen": -0.7934362888336182, |
| "logits/rejected": -0.8030596971511841, |
| "logps/chosen": -304.19854736328125, |
| "logps/rejected": -326.1468811035156, |
| "loss": 0.6335, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.024688338860869408, |
| "rewards/margins": 0.17326180636882782, |
| "rewards/rejected": -0.19795015454292297, |
| "step": 125 |
| }, |
| { |
| "dpo_lambda": 0.986897349357605, |
| "epoch": 0.2638052865741952, |
| "grad_norm": 12.097213815976195, |
| "learning_rate": 4.603133832077953e-07, |
| "logits/chosen": -0.8276994228363037, |
| "logits/rejected": -0.8561450242996216, |
| "logps/chosen": -279.8558044433594, |
| "logps/rejected": -314.57623291015625, |
| "loss": 0.6161, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.027820399031043053, |
| "rewards/margins": 0.24257630109786987, |
| "rewards/rejected": -0.27039670944213867, |
| "step": 126 |
| }, |
| { |
| "dpo_lambda": 0.9867925047874451, |
| "epoch": 0.26589897932478407, |
| "grad_norm": 12.805906049426016, |
| "learning_rate": 4.5931796656116837e-07, |
| "logits/chosen": -0.8533681631088257, |
| "logits/rejected": -0.9015665054321289, |
| "logps/chosen": -276.1855773925781, |
| "logps/rejected": -236.68948364257812, |
| "loss": 0.6133, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.03461135923862457, |
| "rewards/margins": 0.2219991385936737, |
| "rewards/rejected": -0.2566105127334595, |
| "step": 127 |
| }, |
| { |
| "dpo_lambda": 0.9866876006126404, |
| "epoch": 0.2679926720753729, |
| "grad_norm": 13.678592922691685, |
| "learning_rate": 4.5831132482724193e-07, |
| "logits/chosen": -0.7873022556304932, |
| "logits/rejected": -0.8093962669372559, |
| "logps/chosen": -333.6779479980469, |
| "logps/rejected": -326.9761657714844, |
| "loss": 0.6197, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": 0.007106524892151356, |
| "rewards/margins": 0.2661459445953369, |
| "rewards/rejected": -0.25903940200805664, |
| "step": 128 |
| }, |
| { |
| "dpo_lambda": 0.9865828156471252, |
| "epoch": 0.27008636482596177, |
| "grad_norm": 17.49566260823807, |
| "learning_rate": 4.5729351198915705e-07, |
| "logits/chosen": -0.7583063244819641, |
| "logits/rejected": -0.7912325263023376, |
| "logps/chosen": -297.1742248535156, |
| "logps/rejected": -275.2450256347656, |
| "loss": 0.6159, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.051353394985198975, |
| "rewards/margins": 0.2515460252761841, |
| "rewards/rejected": -0.30289942026138306, |
| "step": 129 |
| }, |
| { |
| "dpo_lambda": 0.9864779114723206, |
| "epoch": 0.2721800575765506, |
| "grad_norm": 25.445809464675612, |
| "learning_rate": 4.5626458262912735e-07, |
| "logits/chosen": -0.8182171583175659, |
| "logits/rejected": -0.9255108833312988, |
| "logps/chosen": -311.99383544921875, |
| "logps/rejected": -313.9915771484375, |
| "loss": 0.5597, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.030926331877708435, |
| "rewards/margins": 0.39924901723861694, |
| "rewards/rejected": -0.4301753342151642, |
| "step": 130 |
| }, |
| { |
| "dpo_lambda": 0.9863730669021606, |
| "epoch": 0.27427375032713947, |
| "grad_norm": 26.072654774993698, |
| "learning_rate": 4.5522459192551166e-07, |
| "logits/chosen": -0.9452874660491943, |
| "logits/rejected": -0.9563214778900146, |
| "logps/chosen": -303.7942199707031, |
| "logps/rejected": -345.29248046875, |
| "loss": 0.5636, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.014308175072073936, |
| "rewards/margins": 0.3875415325164795, |
| "rewards/rejected": -0.40184974670410156, |
| "step": 131 |
| }, |
| { |
| "dpo_lambda": 0.9862684011459351, |
| "epoch": 0.27636744307772837, |
| "grad_norm": 10.966972949000139, |
| "learning_rate": 4.541735956498554e-07, |
| "logits/chosen": -0.8571950793266296, |
| "logits/rejected": -0.9126186966896057, |
| "logps/chosen": -292.915283203125, |
| "logps/rejected": -323.6187744140625, |
| "loss": 0.6018, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.16021493077278137, |
| "rewards/margins": 0.1987317055463791, |
| "rewards/rejected": -0.35894662141799927, |
| "step": 132 |
| }, |
| { |
| "dpo_lambda": 0.9861635565757751, |
| "epoch": 0.2784611358283172, |
| "grad_norm": 34.670416720237725, |
| "learning_rate": 4.5311165016389914e-07, |
| "logits/chosen": -0.8171315789222717, |
| "logits/rejected": -0.9207563400268555, |
| "logps/chosen": -342.6995849609375, |
| "logps/rejected": -296.8974609375, |
| "loss": 0.597, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.10333568602800369, |
| "rewards/margins": 0.25441792607307434, |
| "rewards/rejected": -0.35775357484817505, |
| "step": 133 |
| }, |
| { |
| "dpo_lambda": 0.9860587120056152, |
| "epoch": 0.28055482857890607, |
| "grad_norm": 23.394857486137187, |
| "learning_rate": 4.520388124165564e-07, |
| "logits/chosen": -0.8245919942855835, |
| "logits/rejected": -0.8185215592384338, |
| "logps/chosen": -300.1319580078125, |
| "logps/rejected": -297.31475830078125, |
| "loss": 0.5945, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.19418853521347046, |
| "rewards/margins": 0.20099210739135742, |
| "rewards/rejected": -0.3951806426048279, |
| "step": 134 |
| }, |
| { |
| "dpo_lambda": 0.9859538674354553, |
| "epoch": 0.2826485213294949, |
| "grad_norm": 7.718880144493543, |
| "learning_rate": 4.5095513994085974e-07, |
| "logits/chosen": -0.8987231850624084, |
| "logits/rejected": -0.9302247166633606, |
| "logps/chosen": -382.30987548828125, |
| "logps/rejected": -354.4898681640625, |
| "loss": 0.5767, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.1764790415763855, |
| "rewards/margins": 0.21645787358283997, |
| "rewards/rejected": -0.39293694496154785, |
| "step": 135 |
| }, |
| { |
| "dpo_lambda": 0.9858490228652954, |
| "epoch": 0.28474221408008377, |
| "grad_norm": 17.245232915887495, |
| "learning_rate": 4.498606908508753e-07, |
| "logits/chosen": -0.8018785119056702, |
| "logits/rejected": -0.8587301969528198, |
| "logps/chosen": -328.37103271484375, |
| "logps/rejected": -353.09375, |
| "loss": 0.6028, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.25376439094543457, |
| "rewards/margins": 0.2805079221725464, |
| "rewards/rejected": -0.534272313117981, |
| "step": 136 |
| }, |
| { |
| "dpo_lambda": 0.9857443571090698, |
| "epoch": 0.2868359068306726, |
| "grad_norm": 15.123900053540126, |
| "learning_rate": 4.487555238385862e-07, |
| "logits/chosen": -0.7876870632171631, |
| "logits/rejected": -0.8026723861694336, |
| "logps/chosen": -352.7374267578125, |
| "logps/rejected": -343.4482421875, |
| "loss": 0.6128, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.346821665763855, |
| "rewards/margins": 0.20842283964157104, |
| "rewards/rejected": -0.555244505405426, |
| "step": 137 |
| }, |
| { |
| "dpo_lambda": 0.9856394529342651, |
| "epoch": 0.28892959958126146, |
| "grad_norm": 26.727256401161796, |
| "learning_rate": 4.476396981707453e-07, |
| "logits/chosen": -0.8450735807418823, |
| "logits/rejected": -0.8367375135421753, |
| "logps/chosen": -352.42425537109375, |
| "logps/rejected": -333.8371887207031, |
| "loss": 0.6371, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.3087271749973297, |
| "rewards/margins": 0.2823084592819214, |
| "rewards/rejected": -0.5910355448722839, |
| "step": 138 |
| }, |
| { |
| "dpo_lambda": 0.9855346083641052, |
| "epoch": 0.2910232923318503, |
| "grad_norm": 14.563523819252962, |
| "learning_rate": 4.4651327368569684e-07, |
| "logits/chosen": -0.845096230506897, |
| "logits/rejected": -0.8808258771896362, |
| "logps/chosen": -354.4432067871094, |
| "logps/rejected": -384.110595703125, |
| "loss": 0.5936, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.2789064645767212, |
| "rewards/margins": 0.35447943210601807, |
| "rewards/rejected": -0.6333858370780945, |
| "step": 139 |
| }, |
| { |
| "dpo_lambda": 0.9854297637939453, |
| "epoch": 0.29311698508243916, |
| "grad_norm": 31.649857730723095, |
| "learning_rate": 4.453763107901675e-07, |
| "logits/chosen": -0.8781813383102417, |
| "logits/rejected": -0.9100785851478577, |
| "logps/chosen": -329.0050048828125, |
| "logps/rejected": -334.69085693359375, |
| "loss": 0.6038, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.3453499972820282, |
| "rewards/margins": 0.23073923587799072, |
| "rewards/rejected": -0.5760892629623413, |
| "step": 140 |
| }, |
| { |
| "dpo_lambda": 0.9853249192237854, |
| "epoch": 0.295210677833028, |
| "grad_norm": 12.957393357512265, |
| "learning_rate": 4.4422887045602674e-07, |
| "logits/chosen": -0.862395167350769, |
| "logits/rejected": -0.9039013981819153, |
| "logps/chosen": -357.94085693359375, |
| "logps/rejected": -343.2564697265625, |
| "loss": 0.6154, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.4187580347061157, |
| "rewards/margins": 0.26384854316711426, |
| "rewards/rejected": -0.6826066374778748, |
| "step": 141 |
| }, |
| { |
| "dpo_lambda": 0.9852200746536255, |
| "epoch": 0.29730437058361686, |
| "grad_norm": 23.14332525158461, |
| "learning_rate": 4.4307101421701755e-07, |
| "logits/chosen": -0.8050290942192078, |
| "logits/rejected": -0.7742469310760498, |
| "logps/chosen": -327.2164001464844, |
| "logps/rejected": -315.2928466796875, |
| "loss": 0.5857, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.27529165148735046, |
| "rewards/margins": 0.37010371685028076, |
| "rewards/rejected": -0.6453953385353088, |
| "step": 142 |
| }, |
| { |
| "dpo_lambda": 0.9851151704788208, |
| "epoch": 0.2993980633342057, |
| "grad_norm": 30.166424003092107, |
| "learning_rate": 4.419028041654559e-07, |
| "logits/chosen": -0.841701865196228, |
| "logits/rejected": -0.9091044664382935, |
| "logps/chosen": -418.12188720703125, |
| "logps/rejected": -399.4429931640625, |
| "loss": 0.5794, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.28405290842056274, |
| "rewards/margins": 0.3614083528518677, |
| "rewards/rejected": -0.6454612612724304, |
| "step": 143 |
| }, |
| { |
| "dpo_lambda": 0.98501056432724, |
| "epoch": 0.30149175608479456, |
| "grad_norm": 26.814166923520364, |
| "learning_rate": 4.4072430294890166e-07, |
| "logits/chosen": -0.858585000038147, |
| "logits/rejected": -0.9625511765480042, |
| "logps/chosen": -327.3453674316406, |
| "logps/rejected": -345.5610656738281, |
| "loss": 0.5726, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.25485959649086, |
| "rewards/margins": 0.4206133484840393, |
| "rewards/rejected": -0.6754729747772217, |
| "step": 144 |
| }, |
| { |
| "dpo_lambda": 0.9849056601524353, |
| "epoch": 0.3035854488353834, |
| "grad_norm": 33.673895861881924, |
| "learning_rate": 4.395355737667985e-07, |
| "logits/chosen": -0.7440855503082275, |
| "logits/rejected": -0.7877547740936279, |
| "logps/chosen": -297.7661437988281, |
| "logps/rejected": -269.4111633300781, |
| "loss": 0.5726, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.2709321677684784, |
| "rewards/margins": 0.4294186532497406, |
| "rewards/rejected": -0.7003507614135742, |
| "step": 145 |
| }, |
| { |
| "dpo_lambda": 0.9848008155822754, |
| "epoch": 0.30567914158597226, |
| "grad_norm": 14.495187775717087, |
| "learning_rate": 4.3833668036708483e-07, |
| "logits/chosen": -0.8378518223762512, |
| "logits/rejected": -0.9328808784484863, |
| "logps/chosen": -362.03997802734375, |
| "logps/rejected": -315.01885986328125, |
| "loss": 0.6036, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.33166226744651794, |
| "rewards/margins": 0.36146122217178345, |
| "rewards/rejected": -0.6931235194206238, |
| "step": 146 |
| }, |
| { |
| "dpo_lambda": 0.9846959710121155, |
| "epoch": 0.3077728343365611, |
| "grad_norm": 24.93817427447256, |
| "learning_rate": 4.3712768704277524e-07, |
| "logits/chosen": -0.8426337242126465, |
| "logits/rejected": -0.8408235907554626, |
| "logps/chosen": -309.55853271484375, |
| "logps/rejected": -324.4660949707031, |
| "loss": 0.6112, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.31168290972709656, |
| "rewards/margins": 0.2369699627161026, |
| "rewards/rejected": -0.5486528873443604, |
| "step": 147 |
| }, |
| { |
| "dpo_lambda": 0.9845911264419556, |
| "epoch": 0.30986652708714996, |
| "grad_norm": 30.124781935289477, |
| "learning_rate": 4.3590865862851263e-07, |
| "logits/chosen": -0.7106437087059021, |
| "logits/rejected": -0.8285320997238159, |
| "logps/chosen": -318.2632141113281, |
| "logps/rejected": -340.59735107421875, |
| "loss": 0.5777, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.31502288579940796, |
| "rewards/margins": 0.3463708162307739, |
| "rewards/rejected": -0.6613936424255371, |
| "step": 148 |
| }, |
| { |
| "dpo_lambda": 0.98448646068573, |
| "epoch": 0.3119602198377388, |
| "grad_norm": 14.108616126201548, |
| "learning_rate": 4.346796604970912e-07, |
| "logits/chosen": -0.8086866736412048, |
| "logits/rejected": -0.8452584743499756, |
| "logps/chosen": -288.48724365234375, |
| "logps/rejected": -311.91583251953125, |
| "loss": 0.6104, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.21689504384994507, |
| "rewards/margins": 0.3877273499965668, |
| "rewards/rejected": -0.6046223640441895, |
| "step": 149 |
| }, |
| { |
| "dpo_lambda": 0.9843816161155701, |
| "epoch": 0.31405391258832765, |
| "grad_norm": 13.499936499224043, |
| "learning_rate": 4.3344075855595097e-07, |
| "logits/chosen": -0.9197017550468445, |
| "logits/rejected": -0.8716071844100952, |
| "logps/chosen": -272.93463134765625, |
| "logps/rejected": -351.92022705078125, |
| "loss": 0.6296, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.2734215557575226, |
| "rewards/margins": 0.2905915081501007, |
| "rewards/rejected": -0.5640130639076233, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.31405391258832765, |
| "eval_dpo_lambda": 0.9842765927314758, |
| "eval_logits/chosen": -0.8554379940032959, |
| "eval_logits/rejected": -0.9008473753929138, |
| "eval_logps/chosen": -323.7625427246094, |
| "eval_logps/rejected": -323.91766357421875, |
| "eval_loss": 0.5972165465354919, |
| "eval_rewards/accuracies": 0.7099999785423279, |
| "eval_rewards/chosen": -0.23115603625774384, |
| "eval_rewards/margins": 0.2977127432823181, |
| "eval_rewards/rejected": -0.5288687944412231, |
| "eval_runtime": 561.0863, |
| "eval_samples_per_second": 3.565, |
| "eval_steps_per_second": 0.891, |
| "step": 150 |
| }, |
| { |
| "dpo_lambda": 0.9842767119407654, |
| "epoch": 0.3161476053389165, |
| "grad_norm": 12.904660992134803, |
| "learning_rate": 4.3219201924364323e-07, |
| "logits/chosen": -0.8644286394119263, |
| "logits/rejected": -0.9774547815322876, |
| "logps/chosen": -282.9930114746094, |
| "logps/rejected": -282.949462890625, |
| "loss": 0.6005, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.2265167385339737, |
| "rewards/margins": 0.2916352152824402, |
| "rewards/rejected": -0.5181519389152527, |
| "step": 151 |
| }, |
| { |
| "dpo_lambda": 0.9841719269752502, |
| "epoch": 0.31824129808950535, |
| "grad_norm": 12.854949528250394, |
| "learning_rate": 4.309335095262675e-07, |
| "logits/chosen": -0.8625615239143372, |
| "logits/rejected": -0.8919581174850464, |
| "logps/chosen": -322.06402587890625, |
| "logps/rejected": -339.1501159667969, |
| "loss": 0.5788, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.23511157929897308, |
| "rewards/margins": 0.34126943349838257, |
| "rewards/rejected": -0.5763810276985168, |
| "step": 152 |
| }, |
| { |
| "dpo_lambda": 0.9840670228004456, |
| "epoch": 0.3203349908400942, |
| "grad_norm": 17.39339060709683, |
| "learning_rate": 4.2966529689388064e-07, |
| "logits/chosen": -0.7854524850845337, |
| "logits/rejected": -0.8845511674880981, |
| "logps/chosen": -360.5038146972656, |
| "logps/rejected": -352.4007263183594, |
| "loss": 0.5801, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.19930708408355713, |
| "rewards/margins": 0.3281744718551636, |
| "rewards/rejected": -0.5274815559387207, |
| "step": 153 |
| }, |
| { |
| "dpo_lambda": 0.9839621782302856, |
| "epoch": 0.32242868359068305, |
| "grad_norm": 12.230075690240232, |
| "learning_rate": 4.2838744935687716e-07, |
| "logits/chosen": -0.9072690606117249, |
| "logits/rejected": -0.9368817210197449, |
| "logps/chosen": -416.1206359863281, |
| "logps/rejected": -363.433837890625, |
| "loss": 0.5941, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.21174053847789764, |
| "rewards/margins": 0.27678394317626953, |
| "rewards/rejected": -0.48852449655532837, |
| "step": 154 |
| }, |
| { |
| "dpo_lambda": 0.9838575124740601, |
| "epoch": 0.3245223763412719, |
| "grad_norm": 19.713309712130915, |
| "learning_rate": 4.271000354423425e-07, |
| "logits/chosen": -0.8548458814620972, |
| "logits/rejected": -0.8624852299690247, |
| "logps/chosen": -318.42828369140625, |
| "logps/rejected": -347.64312744140625, |
| "loss": 0.5767, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.24433034658432007, |
| "rewards/margins": 0.39910271763801575, |
| "rewards/rejected": -0.6434330344200134, |
| "step": 155 |
| }, |
| { |
| "dpo_lambda": 0.9837526679039001, |
| "epoch": 0.32661606909186075, |
| "grad_norm": 12.298734283978302, |
| "learning_rate": 4.258031241903777e-07, |
| "logits/chosen": -0.864250898361206, |
| "logits/rejected": -0.9269083738327026, |
| "logps/chosen": -321.51031494140625, |
| "logps/rejected": -344.8136291503906, |
| "loss": 0.6089, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.1979214996099472, |
| "rewards/margins": 0.3120463788509369, |
| "rewards/rejected": -0.5099678635597229, |
| "step": 156 |
| }, |
| { |
| "dpo_lambda": 0.9836478233337402, |
| "epoch": 0.3287097618424496, |
| "grad_norm": 14.320512468110298, |
| "learning_rate": 4.2449678515039743e-07, |
| "logits/chosen": -0.7892401218414307, |
| "logits/rejected": -0.8079796433448792, |
| "logps/chosen": -285.41900634765625, |
| "logps/rejected": -318.0777893066406, |
| "loss": 0.5903, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.18783962726593018, |
| "rewards/margins": 0.3478223383426666, |
| "rewards/rejected": -0.5356619954109192, |
| "step": 157 |
| }, |
| { |
| "dpo_lambda": 0.9835429787635803, |
| "epoch": 0.33080345459303845, |
| "grad_norm": 18.475777829614685, |
| "learning_rate": 4.2318108837739986e-07, |
| "logits/chosen": -0.8126506805419922, |
| "logits/rejected": -0.8585901260375977, |
| "logps/chosen": -347.2370910644531, |
| "logps/rejected": -341.1890563964844, |
| "loss": 0.5828, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.23844991624355316, |
| "rewards/margins": 0.38391149044036865, |
| "rewards/rejected": -0.6223613619804382, |
| "step": 158 |
| }, |
| { |
| "dpo_lambda": 0.9834380745887756, |
| "epoch": 0.3328971473436273, |
| "grad_norm": 24.273769149931997, |
| "learning_rate": 4.218561044282098e-07, |
| "logits/chosen": -0.8415407538414001, |
| "logits/rejected": -0.9418197274208069, |
| "logps/chosen": -289.1123046875, |
| "logps/rejected": -292.8475036621094, |
| "loss": 0.603, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.2409561723470688, |
| "rewards/margins": 0.232768252491951, |
| "rewards/rejected": -0.4737243950366974, |
| "step": 159 |
| }, |
| { |
| "dpo_lambda": 0.9833334684371948, |
| "epoch": 0.33499084009421615, |
| "grad_norm": 19.16029439073329, |
| "learning_rate": 4.2052190435769554e-07, |
| "logits/chosen": -0.8843802809715271, |
| "logits/rejected": -0.8616979718208313, |
| "logps/chosen": -278.7174072265625, |
| "logps/rejected": -297.68646240234375, |
| "loss": 0.5729, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.22543789446353912, |
| "rewards/margins": 0.3052656948566437, |
| "rewards/rejected": -0.530703604221344, |
| "step": 160 |
| }, |
| { |
| "dpo_lambda": 0.9832285642623901, |
| "epoch": 0.33708453284480505, |
| "grad_norm": 27.70767136536445, |
| "learning_rate": 4.1917855971495763e-07, |
| "logits/chosen": -0.8649369478225708, |
| "logits/rejected": -0.8855935335159302, |
| "logps/chosen": -345.121337890625, |
| "logps/rejected": -321.77685546875, |
| "loss": 0.5818, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.23856690526008606, |
| "rewards/margins": 0.43436694145202637, |
| "rewards/rejected": -0.6729338765144348, |
| "step": 161 |
| }, |
| { |
| "dpo_lambda": 0.9831237196922302, |
| "epoch": 0.3391782255953939, |
| "grad_norm": 11.16607639247331, |
| "learning_rate": 4.1782614253949255e-07, |
| "logits/chosen": -0.8402847647666931, |
| "logits/rejected": -0.8687560558319092, |
| "logps/chosen": -296.60333251953125, |
| "logps/rejected": -322.4178466796875, |
| "loss": 0.5853, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.23870237171649933, |
| "rewards/margins": 0.29797500371932983, |
| "rewards/rejected": -0.5366774201393127, |
| "step": 162 |
| }, |
| { |
| "dpo_lambda": 0.9830188751220703, |
| "epoch": 0.34127191834598275, |
| "grad_norm": 12.827576377715113, |
| "learning_rate": 4.164647253573289e-07, |
| "logits/chosen": -0.8716577887535095, |
| "logits/rejected": -0.8448370695114136, |
| "logps/chosen": -319.44024658203125, |
| "logps/rejected": -368.4493103027344, |
| "loss": 0.5854, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.18674519658088684, |
| "rewards/margins": 0.33129802346229553, |
| "rewards/rejected": -0.5180432796478271, |
| "step": 163 |
| }, |
| { |
| "dpo_lambda": 0.9829140305519104, |
| "epoch": 0.3433656110965716, |
| "grad_norm": 22.369252142701303, |
| "learning_rate": 4.1509438117713863e-07, |
| "logits/chosen": -0.8533887267112732, |
| "logits/rejected": -0.8673970103263855, |
| "logps/chosen": -253.2539520263672, |
| "logps/rejected": -341.646240234375, |
| "loss": 0.5423, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -0.22686009109020233, |
| "rewards/margins": 0.4404720067977905, |
| "rewards/rejected": -0.667332112789154, |
| "step": 164 |
| }, |
| { |
| "dpo_lambda": 0.9828091859817505, |
| "epoch": 0.34545930384716045, |
| "grad_norm": 28.98060239941749, |
| "learning_rate": 4.137151834863213e-07, |
| "logits/chosen": -0.9048106670379639, |
| "logits/rejected": -0.8988927602767944, |
| "logps/chosen": -297.9773254394531, |
| "logps/rejected": -348.6954040527344, |
| "loss": 0.6144, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.26717424392700195, |
| "rewards/margins": 0.3538207411766052, |
| "rewards/rejected": -0.6209949851036072, |
| "step": 165 |
| }, |
| { |
| "dpo_lambda": 0.9827042818069458, |
| "epoch": 0.3475529965977493, |
| "grad_norm": 31.29118255209507, |
| "learning_rate": 4.123272062470633e-07, |
| "logits/chosen": -0.8654748201370239, |
| "logits/rejected": -0.9055193662643433, |
| "logps/chosen": -285.4839172363281, |
| "logps/rejected": -312.4461364746094, |
| "loss": 0.6107, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.27893736958503723, |
| "rewards/margins": 0.3586209714412689, |
| "rewards/rejected": -0.6375582814216614, |
| "step": 166 |
| }, |
| { |
| "dpo_lambda": 0.9825996160507202, |
| "epoch": 0.34964668934833815, |
| "grad_norm": 24.71676505385136, |
| "learning_rate": 4.1093052389237174e-07, |
| "logits/chosen": -0.8707860708236694, |
| "logits/rejected": -0.915932834148407, |
| "logps/chosen": -295.7852478027344, |
| "logps/rejected": -305.9165954589844, |
| "loss": 0.6194, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.323090136051178, |
| "rewards/margins": 0.2546401023864746, |
| "rewards/rejected": -0.5777302980422974, |
| "step": 167 |
| }, |
| { |
| "dpo_lambda": 0.9824947714805603, |
| "epoch": 0.351740382098927, |
| "grad_norm": 34.53755245562597, |
| "learning_rate": 4.0952521132208267e-07, |
| "logits/chosen": -0.8362221121788025, |
| "logits/rejected": -0.9009256362915039, |
| "logps/chosen": -357.5538024902344, |
| "logps/rejected": -332.5069274902344, |
| "loss": 0.6059, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.3512650430202484, |
| "rewards/margins": 0.34191185235977173, |
| "rewards/rejected": -0.6931769251823425, |
| "step": 168 |
| }, |
| { |
| "dpo_lambda": 0.9823899269104004, |
| "epoch": 0.35383407484951584, |
| "grad_norm": 19.31427143784745, |
| "learning_rate": 4.081113438988443e-07, |
| "logits/chosen": -0.7671630382537842, |
| "logits/rejected": -0.797345757484436, |
| "logps/chosen": -353.42620849609375, |
| "logps/rejected": -344.4584045410156, |
| "loss": 0.6152, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.3698110580444336, |
| "rewards/margins": 0.3182661235332489, |
| "rewards/rejected": -0.6880770921707153, |
| "step": 169 |
| }, |
| { |
| "dpo_lambda": 0.9822850823402405, |
| "epoch": 0.3559277676001047, |
| "grad_norm": 47.091883796618596, |
| "learning_rate": 4.0668899744407567e-07, |
| "logits/chosen": -0.7545144557952881, |
| "logits/rejected": -0.844253659248352, |
| "logps/chosen": -298.80206298828125, |
| "logps/rejected": -371.48785400390625, |
| "loss": 0.5473, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.32974353432655334, |
| "rewards/margins": 0.4235215485095978, |
| "rewards/rejected": -0.7532650828361511, |
| "step": 170 |
| }, |
| { |
| "dpo_lambda": 0.9821802377700806, |
| "epoch": 0.35802146035069354, |
| "grad_norm": 58.75258029657535, |
| "learning_rate": 4.0525824823390043e-07, |
| "logits/chosen": -0.856792151927948, |
| "logits/rejected": -0.8787572383880615, |
| "logps/chosen": -319.53118896484375, |
| "logps/rejected": -321.0688781738281, |
| "loss": 0.5704, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.3029014468193054, |
| "rewards/margins": 0.43716755509376526, |
| "rewards/rejected": -0.7400689721107483, |
| "step": 171 |
| }, |
| { |
| "dpo_lambda": 0.982075572013855, |
| "epoch": 0.3601151531012824, |
| "grad_norm": 13.60799588728662, |
| "learning_rate": 4.0381917299505686e-07, |
| "logits/chosen": -0.8702411651611328, |
| "logits/rejected": -0.9157892465591431, |
| "logps/chosen": -356.87030029296875, |
| "logps/rejected": -373.1155700683594, |
| "loss": 0.5837, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.3470417857170105, |
| "rewards/margins": 0.35888221859931946, |
| "rewards/rejected": -0.7059239745140076, |
| "step": 172 |
| }, |
| { |
| "dpo_lambda": 0.9819707274436951, |
| "epoch": 0.36220884585187124, |
| "grad_norm": 33.81883702967302, |
| "learning_rate": 4.0237184890078243e-07, |
| "logits/chosen": -0.9117413759231567, |
| "logits/rejected": -0.9689619541168213, |
| "logps/chosen": -386.8931579589844, |
| "logps/rejected": -378.5868835449219, |
| "loss": 0.5509, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": -0.4211375415325165, |
| "rewards/margins": 0.35862040519714355, |
| "rewards/rejected": -0.7797579765319824, |
| "step": 173 |
| }, |
| { |
| "dpo_lambda": 0.9818658232688904, |
| "epoch": 0.3643025386024601, |
| "grad_norm": 25.48673031671082, |
| "learning_rate": 4.00916353566676e-07, |
| "logits/chosen": -0.8898088335990906, |
| "logits/rejected": -0.9703992605209351, |
| "logps/chosen": -353.3209228515625, |
| "logps/rejected": -407.6015930175781, |
| "loss": 0.5869, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.4089928865432739, |
| "rewards/margins": 0.37967556715011597, |
| "rewards/rejected": -0.7886685132980347, |
| "step": 174 |
| }, |
| { |
| "dpo_lambda": 0.9817609786987305, |
| "epoch": 0.36639623135304894, |
| "grad_norm": 60.06891288607582, |
| "learning_rate": 3.994527650465352e-07, |
| "logits/chosen": -0.8689991235733032, |
| "logits/rejected": -0.9080208539962769, |
| "logps/chosen": -352.3197021484375, |
| "logps/rejected": -376.54254150390625, |
| "loss": 0.5918, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.34678542613983154, |
| "rewards/margins": 0.4048980474472046, |
| "rewards/rejected": -0.7516834735870361, |
| "step": 175 |
| }, |
| { |
| "dpo_lambda": 0.9816561341285706, |
| "epoch": 0.3684899241036378, |
| "grad_norm": 44.658305155054634, |
| "learning_rate": 3.979811618281705e-07, |
| "logits/chosen": -0.846853494644165, |
| "logits/rejected": -0.8561269044876099, |
| "logps/chosen": -276.9634704589844, |
| "logps/rejected": -319.3321838378906, |
| "loss": 0.541, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.2748173773288727, |
| "rewards/margins": 0.4001079499721527, |
| "rewards/rejected": -0.6749253869056702, |
| "step": 176 |
| }, |
| { |
| "dpo_lambda": 0.9815512895584106, |
| "epoch": 0.37058361685422664, |
| "grad_norm": 76.56965412827621, |
| "learning_rate": 3.9650162282919654e-07, |
| "logits/chosen": -0.7609463930130005, |
| "logits/rejected": -0.8508155941963196, |
| "logps/chosen": -374.0244445800781, |
| "logps/rejected": -343.30615234375, |
| "loss": 0.5848, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.25858065485954285, |
| "rewards/margins": 0.4455993175506592, |
| "rewards/rejected": -0.7041800022125244, |
| "step": 177 |
| }, |
| { |
| "dpo_lambda": 0.9814466238021851, |
| "epoch": 0.3726773096048155, |
| "grad_norm": 23.278575599376282, |
| "learning_rate": 3.9501422739279953e-07, |
| "logits/chosen": -0.879623293876648, |
| "logits/rejected": -0.8756167888641357, |
| "logps/chosen": -329.4707946777344, |
| "logps/rejected": -348.7636413574219, |
| "loss": 0.5543, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.2765873074531555, |
| "rewards/margins": 0.47019055485725403, |
| "rewards/rejected": -0.7467778921127319, |
| "step": 178 |
| }, |
| { |
| "dpo_lambda": 0.9813417792320251, |
| "epoch": 0.37477100235540434, |
| "grad_norm": 21.754414666752645, |
| "learning_rate": 3.935190552834828e-07, |
| "logits/chosen": -0.9142841100692749, |
| "logits/rejected": -0.9394044876098633, |
| "logps/chosen": -328.6793518066406, |
| "logps/rejected": -372.35064697265625, |
| "loss": 0.582, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.2454470992088318, |
| "rewards/margins": 0.38764071464538574, |
| "rewards/rejected": -0.6330878138542175, |
| "step": 179 |
| }, |
| { |
| "dpo_lambda": 0.9812368750572205, |
| "epoch": 0.3768646951059932, |
| "grad_norm": 18.271146036301225, |
| "learning_rate": 3.920161866827889e-07, |
| "logits/chosen": -0.9230740666389465, |
| "logits/rejected": -0.9215205907821655, |
| "logps/chosen": -291.40277099609375, |
| "logps/rejected": -328.21923828125, |
| "loss": 0.5845, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.2792706787586212, |
| "rewards/margins": 0.36451247334480286, |
| "rewards/rejected": -0.6437831521034241, |
| "step": 180 |
| }, |
| { |
| "dpo_lambda": 0.9811320900917053, |
| "epoch": 0.37895838785658204, |
| "grad_norm": 51.1159936417294, |
| "learning_rate": 3.90505702185e-07, |
| "logits/chosen": -0.9331147074699402, |
| "logits/rejected": -0.9082576632499695, |
| "logps/chosen": -305.66302490234375, |
| "logps/rejected": -335.3623046875, |
| "loss": 0.6069, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.41276705265045166, |
| "rewards/margins": 0.3198656439781189, |
| "rewards/rejected": -0.7326326966285706, |
| "step": 181 |
| }, |
| { |
| "dpo_lambda": 0.9810271859169006, |
| "epoch": 0.3810520806071709, |
| "grad_norm": 32.33011835733113, |
| "learning_rate": 3.889876827928156e-07, |
| "logits/chosen": -0.9150969982147217, |
| "logits/rejected": -0.8999711275100708, |
| "logps/chosen": -335.1192932128906, |
| "logps/rejected": -358.4149475097656, |
| "loss": 0.5721, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.20234210789203644, |
| "rewards/margins": 0.4882466793060303, |
| "rewards/rejected": -0.6905888319015503, |
| "step": 182 |
| }, |
| { |
| "dpo_lambda": 0.9809223413467407, |
| "epoch": 0.38314577335775973, |
| "grad_norm": 22.410780663539438, |
| "learning_rate": 3.874622099130087e-07, |
| "logits/chosen": -0.8482258319854736, |
| "logits/rejected": -0.8427585363388062, |
| "logps/chosen": -368.4677734375, |
| "logps/rejected": -365.698486328125, |
| "loss": 0.5342, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.31965532898902893, |
| "rewards/margins": 0.5097999572753906, |
| "rewards/rejected": -0.8294552564620972, |
| "step": 183 |
| }, |
| { |
| "dpo_lambda": 0.9808176755905151, |
| "epoch": 0.3852394661083486, |
| "grad_norm": 40.02939969290002, |
| "learning_rate": 3.859293653520604e-07, |
| "logits/chosen": -0.8177944421768188, |
| "logits/rejected": -0.8422019481658936, |
| "logps/chosen": -315.3299865722656, |
| "logps/rejected": -358.39556884765625, |
| "loss": 0.5567, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.24775481224060059, |
| "rewards/margins": 0.4265044331550598, |
| "rewards/rejected": -0.6742592453956604, |
| "step": 184 |
| }, |
| { |
| "dpo_lambda": 0.9807128310203552, |
| "epoch": 0.38733315885893743, |
| "grad_norm": 28.798964377153162, |
| "learning_rate": 3.8438923131177237e-07, |
| "logits/chosen": -0.8667696714401245, |
| "logits/rejected": -0.9194180369377136, |
| "logps/chosen": -323.02752685546875, |
| "logps/rejected": -272.7005615234375, |
| "loss": 0.5917, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.28678402304649353, |
| "rewards/margins": 0.456429660320282, |
| "rewards/rejected": -0.7432136535644531, |
| "step": 185 |
| }, |
| { |
| "dpo_lambda": 0.9806079864501953, |
| "epoch": 0.3894268516095263, |
| "grad_norm": 45.016534861006626, |
| "learning_rate": 3.828418903848593e-07, |
| "logits/chosen": -0.9506804943084717, |
| "logits/rejected": -0.9454731345176697, |
| "logps/chosen": -313.6388244628906, |
| "logps/rejected": -321.62261962890625, |
| "loss": 0.5862, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": -0.4552803635597229, |
| "rewards/margins": 0.20712202787399292, |
| "rewards/rejected": -0.6624024510383606, |
| "step": 186 |
| }, |
| { |
| "dpo_lambda": 0.9805031418800354, |
| "epoch": 0.39152054436011513, |
| "grad_norm": 57.520777558231316, |
| "learning_rate": 3.812874255505191e-07, |
| "logits/chosen": -0.8905811905860901, |
| "logits/rejected": -0.9345070123672485, |
| "logps/chosen": -321.8720703125, |
| "logps/rejected": -344.01934814453125, |
| "loss": 0.5348, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.35681191086769104, |
| "rewards/margins": 0.4626830220222473, |
| "rewards/rejected": -0.819494903087616, |
| "step": 187 |
| }, |
| { |
| "dpo_lambda": 0.9803982377052307, |
| "epoch": 0.393614237110704, |
| "grad_norm": 24.238007823715385, |
| "learning_rate": 3.797259201699833e-07, |
| "logits/chosen": -0.9128227233886719, |
| "logits/rejected": -0.8823633193969727, |
| "logps/chosen": -353.4268493652344, |
| "logps/rejected": -359.056396484375, |
| "loss": 0.6241, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.5055346488952637, |
| "rewards/margins": 0.16114425659179688, |
| "rewards/rejected": -0.6666789054870605, |
| "step": 188 |
| }, |
| { |
| "dpo_lambda": 0.9802933931350708, |
| "epoch": 0.39570792986129283, |
| "grad_norm": 48.17875296602705, |
| "learning_rate": 3.781574579820464e-07, |
| "logits/chosen": -0.8731621503829956, |
| "logits/rejected": -0.8945026397705078, |
| "logps/chosen": -283.6103820800781, |
| "logps/rejected": -288.29144287109375, |
| "loss": 0.6123, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.523615837097168, |
| "rewards/margins": 0.24947702884674072, |
| "rewards/rejected": -0.7730928659439087, |
| "step": 189 |
| }, |
| { |
| "dpo_lambda": 0.9801887273788452, |
| "epoch": 0.39780162261188173, |
| "grad_norm": 18.0718217557433, |
| "learning_rate": 3.765821230985757e-07, |
| "logits/chosen": -0.8426337242126465, |
| "logits/rejected": -0.8446739912033081, |
| "logps/chosen": -367.47344970703125, |
| "logps/rejected": -384.815185546875, |
| "loss": 0.5439, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.39618077874183655, |
| "rewards/margins": 0.5162340998649597, |
| "rewards/rejected": -0.9124149084091187, |
| "step": 190 |
| }, |
| { |
| "dpo_lambda": 0.9800838828086853, |
| "epoch": 0.3998953153624706, |
| "grad_norm": 21.830819632208076, |
| "learning_rate": 3.75e-07, |
| "logits/chosen": -0.9197947978973389, |
| "logits/rejected": -0.9434197545051575, |
| "logps/chosen": -327.9869384765625, |
| "logps/rejected": -383.2161560058594, |
| "loss": 0.6243, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.4305557608604431, |
| "rewards/margins": 0.25802311301231384, |
| "rewards/rejected": -0.6885788440704346, |
| "step": 191 |
| }, |
| { |
| "dpo_lambda": 0.9799790382385254, |
| "epoch": 0.40198900811305943, |
| "grad_norm": 78.10755969507458, |
| "learning_rate": 3.734111735307796e-07, |
| "logits/chosen": -0.8385262489318848, |
| "logits/rejected": -0.8365335464477539, |
| "logps/chosen": -269.979736328125, |
| "logps/rejected": -328.7428283691406, |
| "loss": 0.5734, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.3273383378982544, |
| "rewards/margins": 0.37994542717933655, |
| "rewards/rejected": -0.7072837352752686, |
| "step": 192 |
| }, |
| { |
| "dpo_lambda": 0.9798741936683655, |
| "epoch": 0.4040827008636483, |
| "grad_norm": 13.276534069096611, |
| "learning_rate": 3.7181572889485623e-07, |
| "logits/chosen": -0.8739302158355713, |
| "logits/rejected": -0.8805893659591675, |
| "logps/chosen": -382.5960998535156, |
| "logps/rejected": -386.1152648925781, |
| "loss": 0.5846, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.5214099287986755, |
| "rewards/margins": 0.38677844405174255, |
| "rewards/rejected": -0.9081884622573853, |
| "step": 193 |
| }, |
| { |
| "dpo_lambda": 0.9797693490982056, |
| "epoch": 0.40617639361423713, |
| "grad_norm": 31.90170636280833, |
| "learning_rate": 3.7021375165108377e-07, |
| "logits/chosen": -0.8950821757316589, |
| "logits/rejected": -0.8792182803153992, |
| "logps/chosen": -327.24017333984375, |
| "logps/rejected": -352.606689453125, |
| "loss": 0.5566, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.4424905478954315, |
| "rewards/margins": 0.2954648733139038, |
| "rewards/rejected": -0.7379554510116577, |
| "step": 194 |
| }, |
| { |
| "dpo_lambda": 0.97966468334198, |
| "epoch": 0.408270086364826, |
| "grad_norm": 48.50857855939344, |
| "learning_rate": 3.6860532770864005e-07, |
| "logits/chosen": -0.7950228452682495, |
| "logits/rejected": -0.9293493628501892, |
| "logps/chosen": -341.1165466308594, |
| "logps/rejected": -340.11322021484375, |
| "loss": 0.5936, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.47899454832077026, |
| "rewards/margins": 0.3304406404495239, |
| "rewards/rejected": -0.8094351291656494, |
| "step": 195 |
| }, |
| { |
| "dpo_lambda": 0.9795597791671753, |
| "epoch": 0.41036377911541483, |
| "grad_norm": 14.344018616235443, |
| "learning_rate": 3.6699054332241985e-07, |
| "logits/chosen": -0.9165274500846863, |
| "logits/rejected": -0.948397696018219, |
| "logps/chosen": -339.11407470703125, |
| "logps/rejected": -363.7457275390625, |
| "loss": 0.5414, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.2974359691143036, |
| "rewards/margins": 0.4967215061187744, |
| "rewards/rejected": -0.7941575050354004, |
| "step": 196 |
| }, |
| { |
| "dpo_lambda": 0.9794549345970154, |
| "epoch": 0.4124574718660037, |
| "grad_norm": 16.146850927636255, |
| "learning_rate": 3.653694850884091e-07, |
| "logits/chosen": -0.9266453981399536, |
| "logits/rejected": -0.9306536912918091, |
| "logps/chosen": -336.86529541015625, |
| "logps/rejected": -395.9813537597656, |
| "loss": 0.6083, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.34124135971069336, |
| "rewards/margins": 0.3672024607658386, |
| "rewards/rejected": -0.708443820476532, |
| "step": 197 |
| }, |
| { |
| "dpo_lambda": 0.9793500900268555, |
| "epoch": 0.4145511646165925, |
| "grad_norm": 16.33656205714644, |
| "learning_rate": 3.6374223993904124e-07, |
| "logits/chosen": -0.8765822052955627, |
| "logits/rejected": -0.9160308837890625, |
| "logps/chosen": -349.8119201660156, |
| "logps/rejected": -395.2640380859375, |
| "loss": 0.5783, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.4447760283946991, |
| "rewards/margins": 0.34209224581718445, |
| "rewards/rejected": -0.7868682146072388, |
| "step": 198 |
| }, |
| { |
| "dpo_lambda": 0.9792452454566956, |
| "epoch": 0.4166448573671814, |
| "grad_norm": 23.060842987392316, |
| "learning_rate": 3.621088951385353e-07, |
| "logits/chosen": -0.9447203278541565, |
| "logits/rejected": -0.9942541718482971, |
| "logps/chosen": -333.0228271484375, |
| "logps/rejected": -373.6483459472656, |
| "loss": 0.5973, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.4236346483230591, |
| "rewards/margins": 0.3587067723274231, |
| "rewards/rejected": -0.7823415398597717, |
| "step": 199 |
| }, |
| { |
| "dpo_lambda": 0.9791404008865356, |
| "epoch": 0.4187385501177702, |
| "grad_norm": 35.20357050968758, |
| "learning_rate": 3.604695382782159e-07, |
| "logits/chosen": -0.8524163365364075, |
| "logits/rejected": -0.9107453227043152, |
| "logps/chosen": -377.02886962890625, |
| "logps/rejected": -383.6340026855469, |
| "loss": 0.6219, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.5846968293190002, |
| "rewards/margins": 0.27121788263320923, |
| "rewards/rejected": -0.8559147119522095, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.4187385501177702, |
| "eval_dpo_lambda": 0.9790353775024414, |
| "eval_logits/chosen": -0.8926590085029602, |
| "eval_logits/rejected": -0.9312657117843628, |
| "eval_logps/chosen": -341.60223388671875, |
| "eval_logps/rejected": -351.5380554199219, |
| "eval_loss": 0.578376829624176, |
| "eval_rewards/accuracies": 0.7310000061988831, |
| "eval_rewards/chosen": -0.40955302119255066, |
| "eval_rewards/margins": 0.3955199420452118, |
| "eval_rewards/rejected": -0.8050729632377625, |
| "eval_runtime": 561.0371, |
| "eval_samples_per_second": 3.565, |
| "eval_steps_per_second": 0.891, |
| "step": 200 |
| }, |
| { |
| "dpo_lambda": 0.979035496711731, |
| "epoch": 0.4208322428683591, |
| "grad_norm": 12.619078496504596, |
| "learning_rate": 3.588242572718162e-07, |
| "logits/chosen": -0.81094890832901, |
| "logits/rejected": -0.8716937303543091, |
| "logps/chosen": -346.4378967285156, |
| "logps/rejected": -399.423095703125, |
| "loss": 0.5737, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.3493123948574066, |
| "rewards/margins": 0.4185783267021179, |
| "rewards/rejected": -0.7678907513618469, |
| "step": 201 |
| }, |
| { |
| "dpo_lambda": 0.9789308905601501, |
| "epoch": 0.4229259356189479, |
| "grad_norm": 55.03692628232677, |
| "learning_rate": 3.571731403507635e-07, |
| "logits/chosen": -0.877128005027771, |
| "logits/rejected": -0.8727900385856628, |
| "logps/chosen": -308.8694152832031, |
| "logps/rejected": -371.779296875, |
| "loss": 0.5829, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.4663529098033905, |
| "rewards/margins": 0.43134671449661255, |
| "rewards/rejected": -0.8976995944976807, |
| "step": 202 |
| }, |
| { |
| "dpo_lambda": 0.9788259863853455, |
| "epoch": 0.4250196283695368, |
| "grad_norm": 23.62457430224331, |
| "learning_rate": 3.5551627605944746e-07, |
| "logits/chosen": -0.9041829109191895, |
| "logits/rejected": -0.9442777633666992, |
| "logps/chosen": -306.6086120605469, |
| "logps/rejected": -325.716796875, |
| "loss": 0.6029, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.4549506902694702, |
| "rewards/margins": 0.29961732029914856, |
| "rewards/rejected": -0.7545679807662964, |
| "step": 203 |
| }, |
| { |
| "dpo_lambda": 0.9787211418151855, |
| "epoch": 0.4271133211201256, |
| "grad_norm": 22.05233559020067, |
| "learning_rate": 3.5385375325047163e-07, |
| "logits/chosen": -0.9502891302108765, |
| "logits/rejected": -0.9089999198913574, |
| "logps/chosen": -298.177978515625, |
| "logps/rejected": -349.14715576171875, |
| "loss": 0.6196, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.45129668712615967, |
| "rewards/margins": 0.33733218908309937, |
| "rewards/rejected": -0.7886289358139038, |
| "step": 204 |
| }, |
| { |
| "dpo_lambda": 0.9786162972450256, |
| "epoch": 0.42920701387071447, |
| "grad_norm": 24.065121080878257, |
| "learning_rate": 3.5218566107988867e-07, |
| "logits/chosen": -0.8696624636650085, |
| "logits/rejected": -0.9973980784416199, |
| "logps/chosen": -336.4361877441406, |
| "logps/rejected": -342.7047119140625, |
| "loss": 0.5722, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.4459235370159149, |
| "rewards/margins": 0.40657177567481995, |
| "rewards/rejected": -0.8524953126907349, |
| "step": 205 |
| }, |
| { |
| "dpo_lambda": 0.9785114526748657, |
| "epoch": 0.4313007066213033, |
| "grad_norm": 22.47093848456195, |
| "learning_rate": 3.505120890024195e-07, |
| "logits/chosen": -0.7719739675521851, |
| "logits/rejected": -0.8493779897689819, |
| "logps/chosen": -291.58270263671875, |
| "logps/rejected": -316.1316223144531, |
| "loss": 0.5942, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.36826789379119873, |
| "rewards/margins": 0.37382954359054565, |
| "rewards/rejected": -0.7420974969863892, |
| "step": 206 |
| }, |
| { |
| "dpo_lambda": 0.9784067869186401, |
| "epoch": 0.43339439937189217, |
| "grad_norm": 44.05441694450149, |
| "learning_rate": 3.4883312676665534e-07, |
| "logits/chosen": -0.9384758472442627, |
| "logits/rejected": -0.9461789131164551, |
| "logps/chosen": -323.506591796875, |
| "logps/rejected": -331.2817687988281, |
| "loss": 0.6098, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.3830246329307556, |
| "rewards/margins": 0.41553816199302673, |
| "rewards/rejected": -0.79856276512146, |
| "step": 207 |
| }, |
| { |
| "dpo_lambda": 0.9783019423484802, |
| "epoch": 0.435488092122481, |
| "grad_norm": 26.81934954378791, |
| "learning_rate": 3.4714886441024573e-07, |
| "logits/chosen": -0.9290968179702759, |
| "logits/rejected": -0.9625118374824524, |
| "logps/chosen": -313.3687744140625, |
| "logps/rejected": -333.42425537109375, |
| "loss": 0.5725, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.4035572409629822, |
| "rewards/margins": 0.3620757460594177, |
| "rewards/rejected": -0.7656329870223999, |
| "step": 208 |
| }, |
| { |
| "dpo_lambda": 0.9781970381736755, |
| "epoch": 0.43758178487306987, |
| "grad_norm": 29.672471588932904, |
| "learning_rate": 3.454593922550693e-07, |
| "logits/chosen": -0.9330939054489136, |
| "logits/rejected": -0.9976711869239807, |
| "logps/chosen": -343.06463623046875, |
| "logps/rejected": -349.95062255859375, |
| "loss": 0.6026, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.4826650619506836, |
| "rewards/margins": 0.24588173627853394, |
| "rewards/rejected": -0.7285467982292175, |
| "step": 209 |
| }, |
| { |
| "dpo_lambda": 0.9780922532081604, |
| "epoch": 0.4396754776236587, |
| "grad_norm": 31.246108954505512, |
| "learning_rate": 3.4376480090239047e-07, |
| "logits/chosen": -0.8667219281196594, |
| "logits/rejected": -0.9327815771102905, |
| "logps/chosen": -344.1031799316406, |
| "logps/rejected": -346.087158203125, |
| "loss": 0.5434, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.3688032627105713, |
| "rewards/margins": 0.4194371700286865, |
| "rewards/rejected": -0.7882405519485474, |
| "step": 210 |
| }, |
| { |
| "dpo_lambda": 0.9779873490333557, |
| "epoch": 0.44176917037424757, |
| "grad_norm": 8.990953515181312, |
| "learning_rate": 3.4206518122800055e-07, |
| "logits/chosen": -0.9713053703308105, |
| "logits/rejected": -1.0429770946502686, |
| "logps/chosen": -408.6382751464844, |
| "logps/rejected": -343.381591796875, |
| "loss": 0.5813, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.3207278251647949, |
| "rewards/margins": 0.327059268951416, |
| "rewards/rejected": -0.6477870941162109, |
| "step": 211 |
| }, |
| { |
| "dpo_lambda": 0.9778825044631958, |
| "epoch": 0.4438628631248364, |
| "grad_norm": 23.5993671623435, |
| "learning_rate": 3.403606243773448e-07, |
| "logits/chosen": -0.8349948525428772, |
| "logits/rejected": -0.8876362442970276, |
| "logps/chosen": -308.45355224609375, |
| "logps/rejected": -344.296875, |
| "loss": 0.5984, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.43563535809516907, |
| "rewards/margins": 0.35835322737693787, |
| "rewards/rejected": -0.7939885854721069, |
| "step": 212 |
| }, |
| { |
| "dpo_lambda": 0.9777778387069702, |
| "epoch": 0.44595655587542526, |
| "grad_norm": 31.422298138505585, |
| "learning_rate": 3.3865122176063385e-07, |
| "logits/chosen": -0.8766756057739258, |
| "logits/rejected": -0.9191992282867432, |
| "logps/chosen": -303.934814453125, |
| "logps/rejected": -390.5749816894531, |
| "loss": 0.5448, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.3933745324611664, |
| "rewards/margins": 0.47594064474105835, |
| "rewards/rejected": -0.8693151473999023, |
| "step": 213 |
| }, |
| { |
| "dpo_lambda": 0.9776729941368103, |
| "epoch": 0.4480502486260141, |
| "grad_norm": 17.891900661691857, |
| "learning_rate": 3.3693706504794243e-07, |
| "logits/chosen": -0.8904320001602173, |
| "logits/rejected": -0.8882500529289246, |
| "logps/chosen": -349.6669921875, |
| "logps/rejected": -394.34912109375, |
| "loss": 0.5758, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.4451632499694824, |
| "rewards/margins": 0.2383812963962555, |
| "rewards/rejected": -0.6835446357727051, |
| "step": 214 |
| }, |
| { |
| "dpo_lambda": 0.9775681495666504, |
| "epoch": 0.45014394137660296, |
| "grad_norm": 26.962413321674106, |
| "learning_rate": 3.3521824616429284e-07, |
| "logits/chosen": -0.9297071695327759, |
| "logits/rejected": -1.0011214017868042, |
| "logps/chosen": -299.71392822265625, |
| "logps/rejected": -358.0120544433594, |
| "loss": 0.5707, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.36456355452537537, |
| "rewards/margins": 0.4526520371437073, |
| "rewards/rejected": -0.8172155618667603, |
| "step": 215 |
| }, |
| { |
| "dpo_lambda": 0.9774633049964905, |
| "epoch": 0.4522376341271918, |
| "grad_norm": 19.029499853418418, |
| "learning_rate": 3.334948572847253e-07, |
| "logits/chosen": -0.9595445990562439, |
| "logits/rejected": -0.9602202773094177, |
| "logps/chosen": -287.4220886230469, |
| "logps/rejected": -356.87335205078125, |
| "loss": 0.5705, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.3573736250400543, |
| "rewards/margins": 0.4587840139865875, |
| "rewards/rejected": -0.8161575198173523, |
| "step": 216 |
| }, |
| { |
| "dpo_lambda": 0.9773584008216858, |
| "epoch": 0.45433132687778066, |
| "grad_norm": 42.01861807639346, |
| "learning_rate": 3.317669908293554e-07, |
| "logits/chosen": -0.8164669871330261, |
| "logits/rejected": -0.8533629179000854, |
| "logps/chosen": -361.13067626953125, |
| "logps/rejected": -366.3695068359375, |
| "loss": 0.6027, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.4534375071525574, |
| "rewards/margins": 0.28975534439086914, |
| "rewards/rejected": -0.7431929111480713, |
| "step": 217 |
| }, |
| { |
| "dpo_lambda": 0.977253794670105, |
| "epoch": 0.4564250196283695, |
| "grad_norm": 36.179024344052024, |
| "learning_rate": 3.300347394584172e-07, |
| "logits/chosen": -0.9450358152389526, |
| "logits/rejected": -0.964739203453064, |
| "logps/chosen": -338.791015625, |
| "logps/rejected": -352.18853759765625, |
| "loss": 0.5833, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.509911835193634, |
| "rewards/margins": 0.29347842931747437, |
| "rewards/rejected": -0.8033902049064636, |
| "step": 218 |
| }, |
| { |
| "dpo_lambda": 0.9771488904953003, |
| "epoch": 0.4585187123789584, |
| "grad_norm": 13.931536630842533, |
| "learning_rate": 3.2829819606729477e-07, |
| "logits/chosen": -0.9925215840339661, |
| "logits/rejected": -0.9919613599777222, |
| "logps/chosen": -351.99560546875, |
| "logps/rejected": -336.3402099609375, |
| "loss": 0.5738, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.33854570984840393, |
| "rewards/margins": 0.41257885098457336, |
| "rewards/rejected": -0.7511245012283325, |
| "step": 219 |
| }, |
| { |
| "dpo_lambda": 0.9770440459251404, |
| "epoch": 0.46061240512954726, |
| "grad_norm": 20.796560138343686, |
| "learning_rate": 3.265574537815398e-07, |
| "logits/chosen": -0.9090243577957153, |
| "logits/rejected": -0.87982577085495, |
| "logps/chosen": -291.6847229003906, |
| "logps/rejected": -295.15838623046875, |
| "loss": 0.5522, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.4356476664543152, |
| "rewards/margins": 0.30086103081703186, |
| "rewards/rejected": -0.7365086078643799, |
| "step": 220 |
| }, |
| { |
| "dpo_lambda": 0.9769392013549805, |
| "epoch": 0.4627060978801361, |
| "grad_norm": 17.55086377387192, |
| "learning_rate": 3.248126059518784e-07, |
| "logits/chosen": -0.8062466382980347, |
| "logits/rejected": -0.8155995607376099, |
| "logps/chosen": -350.0935974121094, |
| "logps/rejected": -417.79290771484375, |
| "loss": 0.5654, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.5696654319763184, |
| "rewards/margins": 0.3574390411376953, |
| "rewards/rejected": -0.9271044731140137, |
| "step": 221 |
| }, |
| { |
| "dpo_lambda": 0.9768343567848206, |
| "epoch": 0.46479979063072496, |
| "grad_norm": 21.511465866436165, |
| "learning_rate": 3.230637461492043e-07, |
| "logits/chosen": -0.9671846628189087, |
| "logits/rejected": -0.93825763463974, |
| "logps/chosen": -309.1865234375, |
| "logps/rejected": -365.5205383300781, |
| "loss": 0.5752, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.4525972604751587, |
| "rewards/margins": 0.44703030586242676, |
| "rewards/rejected": -0.8996275663375854, |
| "step": 222 |
| }, |
| { |
| "dpo_lambda": 0.9767295122146606, |
| "epoch": 0.4668934833813138, |
| "grad_norm": 21.781379476586974, |
| "learning_rate": 3.213109681595612e-07, |
| "logits/chosen": -0.885411262512207, |
| "logits/rejected": -0.9873719811439514, |
| "logps/chosen": -344.0679626464844, |
| "logps/rejected": -348.89544677734375, |
| "loss": 0.5736, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.47906970977783203, |
| "rewards/margins": 0.42836642265319824, |
| "rewards/rejected": -0.9074360132217407, |
| "step": 223 |
| }, |
| { |
| "dpo_lambda": 0.976624608039856, |
| "epoch": 0.46898717613190266, |
| "grad_norm": 34.15253461253864, |
| "learning_rate": 3.1955436597911315e-07, |
| "logits/chosen": -0.8723405003547668, |
| "logits/rejected": -0.9412495493888855, |
| "logps/chosen": -360.9906005859375, |
| "logps/rejected": -400.2396240234375, |
| "loss": 0.597, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.5952271223068237, |
| "rewards/margins": 0.4624156057834625, |
| "rewards/rejected": -1.0576426982879639, |
| "step": 224 |
| }, |
| { |
| "dpo_lambda": 0.9765199422836304, |
| "epoch": 0.4710808688824915, |
| "grad_norm": 17.76135160440879, |
| "learning_rate": 3.1779403380910425e-07, |
| "logits/chosen": -0.8463708162307739, |
| "logits/rejected": -0.8820710182189941, |
| "logps/chosen": -297.9212646484375, |
| "logps/rejected": -382.1929931640625, |
| "loss": 0.5682, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.5124987363815308, |
| "rewards/margins": 0.5348488688468933, |
| "rewards/rejected": -1.0473475456237793, |
| "step": 225 |
| }, |
| { |
| "dpo_lambda": 0.9764150977134705, |
| "epoch": 0.47317456163308036, |
| "grad_norm": 27.616034119162922, |
| "learning_rate": 3.160300660508064e-07, |
| "logits/chosen": -0.9552580714225769, |
| "logits/rejected": -0.9441887140274048, |
| "logps/chosen": -325.70379638671875, |
| "logps/rejected": -377.0942687988281, |
| "loss": 0.5982, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.5950958728790283, |
| "rewards/margins": 0.43342557549476624, |
| "rewards/rejected": -1.0285214185714722, |
| "step": 226 |
| }, |
| { |
| "dpo_lambda": 0.9763102531433105, |
| "epoch": 0.4752682543836692, |
| "grad_norm": 35.0109328334649, |
| "learning_rate": 3.1426255730045695e-07, |
| "logits/chosen": -0.9298887252807617, |
| "logits/rejected": -0.8998876214027405, |
| "logps/chosen": -342.6550598144531, |
| "logps/rejected": -410.9444885253906, |
| "loss": 0.5938, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.7437422275543213, |
| "rewards/margins": 0.36158329248428345, |
| "rewards/rejected": -1.10532546043396, |
| "step": 227 |
| }, |
| { |
| "dpo_lambda": 0.9762054085731506, |
| "epoch": 0.47736194713425806, |
| "grad_norm": 21.706855446762994, |
| "learning_rate": 3.1249160234418644e-07, |
| "logits/chosen": -0.9593175649642944, |
| "logits/rejected": -0.9820206165313721, |
| "logps/chosen": -393.0712890625, |
| "logps/rejected": -404.17303466796875, |
| "loss": 0.6354, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.6769300699234009, |
| "rewards/margins": 0.3926386535167694, |
| "rewards/rejected": -1.0695687532424927, |
| "step": 228 |
| }, |
| { |
| "dpo_lambda": 0.9761005640029907, |
| "epoch": 0.4794556398848469, |
| "grad_norm": 50.00541063078205, |
| "learning_rate": 3.1071729615293424e-07, |
| "logits/chosen": -0.9064358472824097, |
| "logits/rejected": -0.9524105787277222, |
| "logps/chosen": -417.2607727050781, |
| "logps/rejected": -373.5041198730469, |
| "loss": 0.6075, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.8266244530677795, |
| "rewards/margins": 0.33187851309776306, |
| "rewards/rejected": -1.1585030555725098, |
| "step": 229 |
| }, |
| { |
| "dpo_lambda": 0.9759958982467651, |
| "epoch": 0.48154933263543576, |
| "grad_norm": 14.21994582685636, |
| "learning_rate": 3.0893973387735683e-07, |
| "logits/chosen": -0.9525047540664673, |
| "logits/rejected": -0.9384512901306152, |
| "logps/chosen": -357.6230163574219, |
| "logps/rejected": -417.1165771484375, |
| "loss": 0.6028, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.6341197490692139, |
| "rewards/margins": 0.4427669644355774, |
| "rewards/rejected": -1.0768866539001465, |
| "step": 230 |
| }, |
| { |
| "dpo_lambda": 0.9758910536766052, |
| "epoch": 0.4836430253860246, |
| "grad_norm": 31.64740174231057, |
| "learning_rate": 3.071590108427243e-07, |
| "logits/chosen": -0.8360011577606201, |
| "logits/rejected": -0.9409224987030029, |
| "logps/chosen": -367.2627258300781, |
| "logps/rejected": -375.06622314453125, |
| "loss": 0.5399, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.6678786277770996, |
| "rewards/margins": 0.44707852602005005, |
| "rewards/rejected": -1.1149570941925049, |
| "step": 231 |
| }, |
| { |
| "dpo_lambda": 0.9757861495018005, |
| "epoch": 0.48573671813661345, |
| "grad_norm": 12.32973304776845, |
| "learning_rate": 3.05375222543809e-07, |
| "logits/chosen": -0.9434400796890259, |
| "logits/rejected": -0.9206544756889343, |
| "logps/chosen": -377.33905029296875, |
| "logps/rejected": -420.7717590332031, |
| "loss": 0.571, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.5599082708358765, |
| "rewards/margins": 0.3856970965862274, |
| "rewards/rejected": -0.9456053972244263, |
| "step": 232 |
| }, |
| { |
| "dpo_lambda": 0.9756813645362854, |
| "epoch": 0.4878304108872023, |
| "grad_norm": 29.85350415813659, |
| "learning_rate": 3.035884646397637e-07, |
| "logits/chosen": -0.8815343976020813, |
| "logits/rejected": -0.9104249477386475, |
| "logps/chosen": -394.9241943359375, |
| "logps/rejected": -365.0865783691406, |
| "loss": 0.5728, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.5130780935287476, |
| "rewards/margins": 0.42393574118614197, |
| "rewards/rejected": -0.9370138049125671, |
| "step": 233 |
| }, |
| { |
| "dpo_lambda": 0.9755764603614807, |
| "epoch": 0.48992410363779115, |
| "grad_norm": 28.006756307210736, |
| "learning_rate": 3.017988329489923e-07, |
| "logits/chosen": -0.8777972459793091, |
| "logits/rejected": -0.837729811668396, |
| "logps/chosen": -272.7120666503906, |
| "logps/rejected": -368.10809326171875, |
| "loss": 0.4952, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.3836371600627899, |
| "rewards/margins": 0.5006729960441589, |
| "rewards/rejected": -0.8843101263046265, |
| "step": 234 |
| }, |
| { |
| "dpo_lambda": 0.9754716157913208, |
| "epoch": 0.49201779638838, |
| "grad_norm": 22.112467921438846, |
| "learning_rate": 3.000064234440111e-07, |
| "logits/chosen": -0.9350335597991943, |
| "logits/rejected": -1.0734158754348755, |
| "logps/chosen": -397.2668762207031, |
| "logps/rejected": -365.10101318359375, |
| "loss": 0.5751, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.4139818251132965, |
| "rewards/margins": 0.4701218605041504, |
| "rewards/rejected": -0.8841036558151245, |
| "step": 235 |
| }, |
| { |
| "dpo_lambda": 0.9753669500350952, |
| "epoch": 0.49411148913896885, |
| "grad_norm": 38.42963861882906, |
| "learning_rate": 2.9821133224630223e-07, |
| "logits/chosen": -0.9367507696151733, |
| "logits/rejected": -0.9378839731216431, |
| "logps/chosen": -346.4058532714844, |
| "logps/rejected": -385.244140625, |
| "loss": 0.5592, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.5048272013664246, |
| "rewards/margins": 0.40106263756752014, |
| "rewards/rejected": -0.9058898687362671, |
| "step": 236 |
| }, |
| { |
| "dpo_lambda": 0.9752621054649353, |
| "epoch": 0.4962051818895577, |
| "grad_norm": 20.272542585780734, |
| "learning_rate": 2.964136556211588e-07, |
| "logits/chosen": -1.0191184282302856, |
| "logits/rejected": -1.0410274267196655, |
| "logps/chosen": -351.78411865234375, |
| "logps/rejected": -365.6335144042969, |
| "loss": 0.5576, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.29722094535827637, |
| "rewards/margins": 0.4697399139404297, |
| "rewards/rejected": -0.766960859298706, |
| "step": 237 |
| }, |
| { |
| "dpo_lambda": 0.9751572012901306, |
| "epoch": 0.49829887464014655, |
| "grad_norm": 12.14766408231827, |
| "learning_rate": 2.946134899725226e-07, |
| "logits/chosen": -0.935865044593811, |
| "logits/rejected": -0.974367618560791, |
| "logps/chosen": -348.61761474609375, |
| "logps/rejected": -370.552978515625, |
| "loss": 0.5527, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.35987845063209534, |
| "rewards/margins": 0.5407595634460449, |
| "rewards/rejected": -0.9006379842758179, |
| "step": 238 |
| }, |
| { |
| "dpo_lambda": 0.9750524163246155, |
| "epoch": 0.5003925673907355, |
| "grad_norm": 10.99279566098769, |
| "learning_rate": 2.9281093183781403e-07, |
| "logits/chosen": -0.8710501194000244, |
| "logits/rejected": -0.8785250186920166, |
| "logps/chosen": -268.412109375, |
| "logps/rejected": -303.83709716796875, |
| "loss": 0.5426, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.2745110094547272, |
| "rewards/margins": 0.5189369916915894, |
| "rewards/rejected": -0.7934479713439941, |
| "step": 239 |
| }, |
| { |
| "dpo_lambda": 0.9749475121498108, |
| "epoch": 0.5024862601413242, |
| "grad_norm": 19.68518029818432, |
| "learning_rate": 2.910060778827554e-07, |
| "logits/chosen": -0.9147160053253174, |
| "logits/rejected": -0.9373922944068909, |
| "logps/chosen": -341.3359069824219, |
| "logps/rejected": -412.0687255859375, |
| "loss": 0.5678, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.2783906161785126, |
| "rewards/margins": 0.5757811069488525, |
| "rewards/rejected": -0.8541717529296875, |
| "step": 240 |
| }, |
| { |
| "dpo_lambda": 0.97484290599823, |
| "epoch": 0.5045799528919132, |
| "grad_norm": 22.785011379156558, |
| "learning_rate": 2.891990248961871e-07, |
| "logits/chosen": -0.941616952419281, |
| "logits/rejected": -0.9725069999694824, |
| "logps/chosen": -358.02911376953125, |
| "logps/rejected": -356.3277587890625, |
| "loss": 0.5623, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.3600071966648102, |
| "rewards/margins": 0.49038365483283997, |
| "rewards/rejected": -0.8503908514976501, |
| "step": 241 |
| }, |
| { |
| "dpo_lambda": 0.9747380018234253, |
| "epoch": 0.506673645642502, |
| "grad_norm": 28.10712370816071, |
| "learning_rate": 2.873898697848762e-07, |
| "logits/chosen": -0.9158104658126831, |
| "logits/rejected": -1.0190154314041138, |
| "logps/chosen": -337.3912353515625, |
| "logps/rejected": -319.40380859375, |
| "loss": 0.5311, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.30729302763938904, |
| "rewards/margins": 0.49401095509529114, |
| "rewards/rejected": -0.8013039827346802, |
| "step": 242 |
| }, |
| { |
| "dpo_lambda": 0.9746331572532654, |
| "epoch": 0.5087673383930909, |
| "grad_norm": 30.036231340589893, |
| "learning_rate": 2.8557870956832133e-07, |
| "logits/chosen": -0.9809325337409973, |
| "logits/rejected": -1.0072345733642578, |
| "logps/chosen": -363.1551208496094, |
| "logps/rejected": -405.18438720703125, |
| "loss": 0.6094, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.3382487893104553, |
| "rewards/margins": 0.4237140417098999, |
| "rewards/rejected": -0.7619627714157104, |
| "step": 243 |
| }, |
| { |
| "dpo_lambda": 0.9745283126831055, |
| "epoch": 0.5108610311436796, |
| "grad_norm": 38.498725672458825, |
| "learning_rate": 2.837656413735479e-07, |
| "logits/chosen": -0.9273790717124939, |
| "logits/rejected": -0.9404940009117126, |
| "logps/chosen": -370.77655029296875, |
| "logps/rejected": -379.8350830078125, |
| "loss": 0.552, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.34177640080451965, |
| "rewards/margins": 0.5170974731445312, |
| "rewards/rejected": -0.8588739037513733, |
| "step": 244 |
| }, |
| { |
| "dpo_lambda": 0.9744234681129456, |
| "epoch": 0.5129547238942685, |
| "grad_norm": 26.230890552567814, |
| "learning_rate": 2.8195076242990116e-07, |
| "logits/chosen": -0.8353450298309326, |
| "logits/rejected": -0.8841926455497742, |
| "logps/chosen": -295.8029479980469, |
| "logps/rejected": -348.716552734375, |
| "loss": 0.5904, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.31430384516716003, |
| "rewards/margins": 0.49294087290763855, |
| "rewards/rejected": -0.8072446584701538, |
| "step": 245 |
| }, |
| { |
| "dpo_lambda": 0.9743185639381409, |
| "epoch": 0.5150484166448573, |
| "grad_norm": 20.742157064446065, |
| "learning_rate": 2.801341700638307e-07, |
| "logits/chosen": -1.0004804134368896, |
| "logits/rejected": -1.0661176443099976, |
| "logps/chosen": -306.6329040527344, |
| "logps/rejected": -319.06097412109375, |
| "loss": 0.5881, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.3133764863014221, |
| "rewards/margins": 0.5279111266136169, |
| "rewards/rejected": -0.8412876725196838, |
| "step": 246 |
| }, |
| { |
| "dpo_lambda": 0.974213719367981, |
| "epoch": 0.5171421093954462, |
| "grad_norm": 41.92705112635296, |
| "learning_rate": 2.7831596169367227e-07, |
| "logits/chosen": -1.0456688404083252, |
| "logits/rejected": -0.9775162935256958, |
| "logps/chosen": -349.86138916015625, |
| "logps/rejected": -417.2496337890625, |
| "loss": 0.5622, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.2810763120651245, |
| "rewards/margins": 0.5058405995368958, |
| "rewards/rejected": -0.7869168519973755, |
| "step": 247 |
| }, |
| { |
| "dpo_lambda": 0.9741090536117554, |
| "epoch": 0.519235802146035, |
| "grad_norm": 35.15777219108126, |
| "learning_rate": 2.7649623482442274e-07, |
| "logits/chosen": -0.9570460319519043, |
| "logits/rejected": -0.9240747094154358, |
| "logps/chosen": -279.88751220703125, |
| "logps/rejected": -371.7086486816406, |
| "loss": 0.5803, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.38907331228256226, |
| "rewards/margins": 0.3909260630607605, |
| "rewards/rejected": -0.7799993753433228, |
| "step": 248 |
| }, |
| { |
| "dpo_lambda": 0.9740042090415955, |
| "epoch": 0.521329494896624, |
| "grad_norm": 42.73962957924612, |
| "learning_rate": 2.7467508704251135e-07, |
| "logits/chosen": -0.9482097625732422, |
| "logits/rejected": -0.9136594533920288, |
| "logps/chosen": -295.8553771972656, |
| "logps/rejected": -348.8441162109375, |
| "loss": 0.5374, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.3581140637397766, |
| "rewards/margins": 0.5417485237121582, |
| "rewards/rejected": -0.8998625874519348, |
| "step": 249 |
| }, |
| { |
| "dpo_lambda": 0.9738993644714355, |
| "epoch": 0.5234231876472127, |
| "grad_norm": 42.5375636334624, |
| "learning_rate": 2.7285261601056697e-07, |
| "logits/chosen": -0.9109609127044678, |
| "logits/rejected": -0.9083539247512817, |
| "logps/chosen": -359.6793212890625, |
| "logps/rejected": -369.906494140625, |
| "loss": 0.5738, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.4524061381816864, |
| "rewards/margins": 0.37430381774902344, |
| "rewards/rejected": -0.8267098665237427, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.5234231876472127, |
| "eval_dpo_lambda": 0.97379469871521, |
| "eval_logits/chosen": -0.933293879032135, |
| "eval_logits/rejected": -0.9690828919410706, |
| "eval_logps/chosen": -344.027587890625, |
| "eval_logps/rejected": -359.6706848144531, |
| "eval_loss": 0.5684590339660645, |
| "eval_rewards/accuracies": 0.7260000109672546, |
| "eval_rewards/chosen": -0.43380701541900635, |
| "eval_rewards/margins": 0.4525919556617737, |
| "eval_rewards/rejected": -0.88639897108078, |
| "eval_runtime": 561.0273, |
| "eval_samples_per_second": 3.565, |
| "eval_steps_per_second": 0.891, |
| "step": 250 |
| }, |
| { |
| "dpo_lambda": 0.9737945199012756, |
| "epoch": 0.5255168803978016, |
| "grad_norm": 21.60785466834392, |
| "learning_rate": 2.7102891946217994e-07, |
| "logits/chosen": -0.9251986742019653, |
| "logits/rejected": -0.9310693740844727, |
| "logps/chosen": -336.9014892578125, |
| "logps/rejected": -380.8359069824219, |
| "loss": 0.5422, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.44501379132270813, |
| "rewards/margins": 0.45862460136413574, |
| "rewards/rejected": -0.903638482093811, |
| "step": 251 |
| }, |
| { |
| "dpo_lambda": 0.9736896753311157, |
| "epoch": 0.5276105731483904, |
| "grad_norm": 19.01047964693019, |
| "learning_rate": 2.692040951966617e-07, |
| "logits/chosen": -1.0268956422805786, |
| "logits/rejected": -0.991072416305542, |
| "logps/chosen": -305.11199951171875, |
| "logps/rejected": -360.6181945800781, |
| "loss": 0.575, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.34655502438545227, |
| "rewards/margins": 0.4588647484779358, |
| "rewards/rejected": -0.8054197430610657, |
| "step": 252 |
| }, |
| { |
| "dpo_lambda": 0.9735850095748901, |
| "epoch": 0.5297042658989793, |
| "grad_norm": 22.46505736505972, |
| "learning_rate": 2.6737824107379947e-07, |
| "logits/chosen": -0.9305629730224609, |
| "logits/rejected": -1.0360764265060425, |
| "logps/chosen": -383.3492736816406, |
| "logps/rejected": -389.62158203125, |
| "loss": 0.5413, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.37200137972831726, |
| "rewards/margins": 0.5039467215538025, |
| "rewards/rejected": -0.8759480714797974, |
| "step": 253 |
| }, |
| { |
| "dpo_lambda": 0.9734801650047302, |
| "epoch": 0.5317979586495681, |
| "grad_norm": 23.321068867881976, |
| "learning_rate": 2.655514550086086e-07, |
| "logits/chosen": -1.0165340900421143, |
| "logits/rejected": -1.0736379623413086, |
| "logps/chosen": -300.9527587890625, |
| "logps/rejected": -284.6766052246094, |
| "loss": 0.575, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.3121712803840637, |
| "rewards/margins": 0.4902705252170563, |
| "rewards/rejected": -0.8024417757987976, |
| "step": 254 |
| }, |
| { |
| "dpo_lambda": 0.9733752608299255, |
| "epoch": 0.533891651400157, |
| "grad_norm": 23.03157221916909, |
| "learning_rate": 2.6372383496608186e-07, |
| "logits/chosen": -0.9229246973991394, |
| "logits/rejected": -0.9603086113929749, |
| "logps/chosen": -347.5473937988281, |
| "logps/rejected": -392.124267578125, |
| "loss": 0.5832, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.4431746006011963, |
| "rewards/margins": 0.49735429883003235, |
| "rewards/rejected": -0.9405289888381958, |
| "step": 255 |
| }, |
| { |
| "dpo_lambda": 0.9732704162597656, |
| "epoch": 0.5359853441507458, |
| "grad_norm": 34.91779344587045, |
| "learning_rate": 2.618954789559356e-07, |
| "logits/chosen": -0.9654712677001953, |
| "logits/rejected": -0.950838565826416, |
| "logps/chosen": -334.4616394042969, |
| "logps/rejected": -320.8468017578125, |
| "loss": 0.5851, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.587689995765686, |
| "rewards/margins": 0.39830049872398376, |
| "rewards/rejected": -0.9859905242919922, |
| "step": 256 |
| }, |
| { |
| "dpo_lambda": 0.9731655716896057, |
| "epoch": 0.5380790369013347, |
| "grad_norm": 14.085882529594617, |
| "learning_rate": 2.600664850273538e-07, |
| "logits/chosen": -0.9465386867523193, |
| "logits/rejected": -0.9746992588043213, |
| "logps/chosen": -362.1474609375, |
| "logps/rejected": -368.56982421875, |
| "loss": 0.5371, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.506095290184021, |
| "rewards/margins": 0.5111798644065857, |
| "rewards/rejected": -1.017275094985962, |
| "step": 257 |
| }, |
| { |
| "dpo_lambda": 0.9730607271194458, |
| "epoch": 0.5401727296519235, |
| "grad_norm": 15.014510984375404, |
| "learning_rate": 2.582369512637302e-07, |
| "logits/chosen": -0.9997269511222839, |
| "logits/rejected": -1.0087834596633911, |
| "logps/chosen": -357.7166442871094, |
| "logps/rejected": -397.3079833984375, |
| "loss": 0.5591, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.6475968956947327, |
| "rewards/margins": 0.450656533241272, |
| "rewards/rejected": -1.0982534885406494, |
| "step": 258 |
| }, |
| { |
| "dpo_lambda": 0.9729558229446411, |
| "epoch": 0.5422664224025124, |
| "grad_norm": 48.7863772083964, |
| "learning_rate": 2.5640697577740815e-07, |
| "logits/chosen": -1.0039561986923218, |
| "logits/rejected": -0.9903517365455627, |
| "logps/chosen": -336.15496826171875, |
| "logps/rejected": -392.0978698730469, |
| "loss": 0.5776, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.5708963871002197, |
| "rewards/margins": 0.4194872975349426, |
| "rewards/rejected": -0.9903836846351624, |
| "step": 259 |
| }, |
| { |
| "dpo_lambda": 0.9728512167930603, |
| "epoch": 0.5443601151531012, |
| "grad_norm": 43.91502909416745, |
| "learning_rate": 2.5457665670441937e-07, |
| "logits/chosen": -0.9723465442657471, |
| "logits/rejected": -0.9578585028648376, |
| "logps/chosen": -315.97186279296875, |
| "logps/rejected": -356.4257507324219, |
| "loss": 0.5949, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.4254854619503021, |
| "rewards/margins": 0.569304347038269, |
| "rewards/rejected": -0.994789719581604, |
| "step": 260 |
| }, |
| { |
| "dpo_lambda": 0.9727463126182556, |
| "epoch": 0.5464538079036901, |
| "grad_norm": 34.6980965461914, |
| "learning_rate": 2.527460921992209e-07, |
| "logits/chosen": -0.9857003688812256, |
| "logits/rejected": -0.9960317611694336, |
| "logps/chosen": -363.2552490234375, |
| "logps/rejected": -397.5711975097656, |
| "loss": 0.5132, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.43664252758026123, |
| "rewards/margins": 0.5617834329605103, |
| "rewards/rejected": -0.9984259605407715, |
| "step": 261 |
| }, |
| { |
| "dpo_lambda": 0.9726415276527405, |
| "epoch": 0.5485475006542789, |
| "grad_norm": 15.205720751303948, |
| "learning_rate": 2.509153804294318e-07, |
| "logits/chosen": -0.9044791460037231, |
| "logits/rejected": -0.9335215091705322, |
| "logps/chosen": -326.0076904296875, |
| "logps/rejected": -371.7582092285156, |
| "loss": 0.5265, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.4435957968235016, |
| "rewards/margins": 0.5709205865859985, |
| "rewards/rejected": -1.0145163536071777, |
| "step": 262 |
| }, |
| { |
| "dpo_lambda": 0.9725366234779358, |
| "epoch": 0.5506411934048678, |
| "grad_norm": 43.35266805487184, |
| "learning_rate": 2.4908461957056825e-07, |
| "logits/chosen": -0.9475391507148743, |
| "logits/rejected": -0.9761734008789062, |
| "logps/chosen": -405.1850891113281, |
| "logps/rejected": -402.66412353515625, |
| "loss": 0.5699, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.44304272532463074, |
| "rewards/margins": 0.4835435450077057, |
| "rewards/rejected": -0.9265862107276917, |
| "step": 263 |
| }, |
| { |
| "dpo_lambda": 0.9724317789077759, |
| "epoch": 0.5527348861554567, |
| "grad_norm": 32.41316077566274, |
| "learning_rate": 2.4725390780077905e-07, |
| "logits/chosen": -0.9430174827575684, |
| "logits/rejected": -0.9314047694206238, |
| "logps/chosen": -363.8974914550781, |
| "logps/rejected": -381.5852966308594, |
| "loss": 0.5414, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.5308617353439331, |
| "rewards/margins": 0.37887492775917053, |
| "rewards/rejected": -0.9097366333007812, |
| "step": 264 |
| }, |
| { |
| "dpo_lambda": 0.9723271131515503, |
| "epoch": 0.5548285789060455, |
| "grad_norm": 40.621721822921614, |
| "learning_rate": 2.454233432955807e-07, |
| "logits/chosen": -0.8697124719619751, |
| "logits/rejected": -0.9561302065849304, |
| "logps/chosen": -302.15875244140625, |
| "logps/rejected": -327.24993896484375, |
| "loss": 0.5883, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.45329856872558594, |
| "rewards/margins": 0.38187170028686523, |
| "rewards/rejected": -0.835170328617096, |
| "step": 265 |
| }, |
| { |
| "dpo_lambda": 0.9722222685813904, |
| "epoch": 0.5569222716566344, |
| "grad_norm": 368.844160392225, |
| "learning_rate": 2.435930242225919e-07, |
| "logits/chosen": -1.0060429573059082, |
| "logits/rejected": -0.9610638618469238, |
| "logps/chosen": -361.58856201171875, |
| "logps/rejected": -349.51171875, |
| "loss": 0.5887, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.4895636737346649, |
| "rewards/margins": 0.3743961751461029, |
| "rewards/rejected": -0.863959789276123, |
| "step": 266 |
| }, |
| { |
| "dpo_lambda": 0.9721174240112305, |
| "epoch": 0.5590159644072232, |
| "grad_norm": 28.521813474186093, |
| "learning_rate": 2.4176304873626984e-07, |
| "logits/chosen": -0.9990183115005493, |
| "logits/rejected": -1.001824140548706, |
| "logps/chosen": -289.71588134765625, |
| "logps/rejected": -331.92138671875, |
| "loss": 0.5715, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.42115557193756104, |
| "rewards/margins": 0.5132542848587036, |
| "rewards/rejected": -0.9344099164009094, |
| "step": 267 |
| }, |
| { |
| "dpo_lambda": 0.9720125794410706, |
| "epoch": 0.5611096571578121, |
| "grad_norm": 22.847288152780855, |
| "learning_rate": 2.399335149726463e-07, |
| "logits/chosen": -0.9127358198165894, |
| "logits/rejected": -0.9713425040245056, |
| "logps/chosen": -324.10101318359375, |
| "logps/rejected": -332.75848388671875, |
| "loss": 0.5619, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.3632182478904724, |
| "rewards/margins": 0.5284265279769897, |
| "rewards/rejected": -0.8916447758674622, |
| "step": 268 |
| }, |
| { |
| "dpo_lambda": 0.9719076752662659, |
| "epoch": 0.5632033499084009, |
| "grad_norm": 19.55742024939717, |
| "learning_rate": 2.381045210440644e-07, |
| "logits/chosen": -0.8354206681251526, |
| "logits/rejected": -0.8707402348518372, |
| "logps/chosen": -338.0024719238281, |
| "logps/rejected": -319.0690002441406, |
| "loss": 0.6275, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": -0.5561378002166748, |
| "rewards/margins": 0.29612746834754944, |
| "rewards/rejected": -0.8522651791572571, |
| "step": 269 |
| }, |
| { |
| "dpo_lambda": 0.971802830696106, |
| "epoch": 0.5652970426589898, |
| "grad_norm": 18.91699155034226, |
| "learning_rate": 2.3627616503391812e-07, |
| "logits/chosen": -0.9677299857139587, |
| "logits/rejected": -1.0638877153396606, |
| "logps/chosen": -398.1684875488281, |
| "logps/rejected": -357.58416748046875, |
| "loss": 0.5519, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.32594913244247437, |
| "rewards/margins": 0.5193861126899719, |
| "rewards/rejected": -0.8453351855278015, |
| "step": 270 |
| }, |
| { |
| "dpo_lambda": 0.9716981649398804, |
| "epoch": 0.5673907354095786, |
| "grad_norm": 102.59972259193498, |
| "learning_rate": 2.344485449913914e-07, |
| "logits/chosen": -0.911102831363678, |
| "logits/rejected": -1.0000059604644775, |
| "logps/chosen": -372.7582702636719, |
| "logps/rejected": -373.64996337890625, |
| "loss": 0.5403, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.42270204424858093, |
| "rewards/margins": 0.5330160856246948, |
| "rewards/rejected": -0.9557181596755981, |
| "step": 271 |
| }, |
| { |
| "dpo_lambda": 0.9715933203697205, |
| "epoch": 0.5694844281601675, |
| "grad_norm": 22.287403967915196, |
| "learning_rate": 2.3262175892620062e-07, |
| "logits/chosen": -0.9188674092292786, |
| "logits/rejected": -0.9886456727981567, |
| "logps/chosen": -334.9513244628906, |
| "logps/rejected": -324.5064697265625, |
| "loss": 0.5841, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": -0.5083807110786438, |
| "rewards/margins": 0.40126848220825195, |
| "rewards/rejected": -0.9096491932868958, |
| "step": 272 |
| }, |
| { |
| "dpo_lambda": 0.9714884757995605, |
| "epoch": 0.5715781209107563, |
| "grad_norm": 17.09621759169118, |
| "learning_rate": 2.3079590480333827e-07, |
| "logits/chosen": -0.9051034450531006, |
| "logits/rejected": -0.9472033977508545, |
| "logps/chosen": -316.8356628417969, |
| "logps/rejected": -348.45428466796875, |
| "loss": 0.5844, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.46500861644744873, |
| "rewards/margins": 0.4641115069389343, |
| "rewards/rejected": -0.9291200637817383, |
| "step": 273 |
| }, |
| { |
| "dpo_lambda": 0.9713836312294006, |
| "epoch": 0.5736718136613452, |
| "grad_norm": 26.00890481729291, |
| "learning_rate": 2.2897108053782e-07, |
| "logits/chosen": -1.0062130689620972, |
| "logits/rejected": -1.063223958015442, |
| "logps/chosen": -324.51312255859375, |
| "logps/rejected": -338.21392822265625, |
| "loss": 0.6106, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.3091861605644226, |
| "rewards/margins": 0.3285538852214813, |
| "rewards/rejected": -0.6377400159835815, |
| "step": 274 |
| }, |
| { |
| "dpo_lambda": 0.9712787866592407, |
| "epoch": 0.575765506411934, |
| "grad_norm": 21.297733625729723, |
| "learning_rate": 2.2714738398943308e-07, |
| "logits/chosen": -0.8751081228256226, |
| "logits/rejected": -0.9124285578727722, |
| "logps/chosen": -327.5624084472656, |
| "logps/rejected": -395.2243347167969, |
| "loss": 0.5115, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.24274717271327972, |
| "rewards/margins": 0.703201413154602, |
| "rewards/rejected": -0.9459485411643982, |
| "step": 275 |
| }, |
| { |
| "dpo_lambda": 0.9711741209030151, |
| "epoch": 0.5778591991625229, |
| "grad_norm": 42.009923318640546, |
| "learning_rate": 2.2532491295748865e-07, |
| "logits/chosen": -1.0958585739135742, |
| "logits/rejected": -1.0163934230804443, |
| "logps/chosen": -331.8833923339844, |
| "logps/rejected": -386.0252380371094, |
| "loss": 0.5985, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.3500264585018158, |
| "rewards/margins": 0.35328248143196106, |
| "rewards/rejected": -0.7033089399337769, |
| "step": 276 |
| }, |
| { |
| "dpo_lambda": 0.9710692167282104, |
| "epoch": 0.5799528919131117, |
| "grad_norm": 24.909203298179698, |
| "learning_rate": 2.2350376517557726e-07, |
| "logits/chosen": -0.9364175200462341, |
| "logits/rejected": -0.9506851434707642, |
| "logps/chosen": -374.86480712890625, |
| "logps/rejected": -346.6588134765625, |
| "loss": 0.6034, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.38636910915374756, |
| "rewards/margins": 0.2424267828464508, |
| "rewards/rejected": -0.628795862197876, |
| "step": 277 |
| }, |
| { |
| "dpo_lambda": 0.9709643721580505, |
| "epoch": 0.5820465846637006, |
| "grad_norm": 17.112568138221835, |
| "learning_rate": 2.2168403830632769e-07, |
| "logits/chosen": -0.9554729461669922, |
| "logits/rejected": -0.9818132519721985, |
| "logps/chosen": -345.8843078613281, |
| "logps/rejected": -395.644775390625, |
| "loss": 0.5531, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.4023358225822449, |
| "rewards/margins": 0.5377238392829895, |
| "rewards/rejected": -0.9400597214698792, |
| "step": 278 |
| }, |
| { |
| "dpo_lambda": 0.9708595275878906, |
| "epoch": 0.5841402774142894, |
| "grad_norm": 27.27451330453449, |
| "learning_rate": 2.1986582993616925e-07, |
| "logits/chosen": -0.8535746335983276, |
| "logits/rejected": -0.9038280248641968, |
| "logps/chosen": -400.73455810546875, |
| "logps/rejected": -396.37921142578125, |
| "loss": 0.5588, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.2820463180541992, |
| "rewards/margins": 0.48483264446258545, |
| "rewards/rejected": -0.7668789029121399, |
| "step": 279 |
| }, |
| { |
| "dpo_lambda": 0.9707546830177307, |
| "epoch": 0.5862339701648783, |
| "grad_norm": 35.46381152183321, |
| "learning_rate": 2.1804923757009882e-07, |
| "logits/chosen": -1.0355491638183594, |
| "logits/rejected": -1.008597493171692, |
| "logps/chosen": -331.9897766113281, |
| "logps/rejected": -345.2893981933594, |
| "loss": 0.6156, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.3384499251842499, |
| "rewards/margins": 0.22750774025917053, |
| "rewards/rejected": -0.5659576654434204, |
| "step": 280 |
| }, |
| { |
| "dpo_lambda": 0.9706498384475708, |
| "epoch": 0.5883276629154671, |
| "grad_norm": 18.095530899983487, |
| "learning_rate": 2.1623435862645205e-07, |
| "logits/chosen": -0.896825909614563, |
| "logits/rejected": -0.8690459728240967, |
| "logps/chosen": -325.27313232421875, |
| "logps/rejected": -389.0777587890625, |
| "loss": 0.5353, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.17332224547863007, |
| "rewards/margins": 0.4838894009590149, |
| "rewards/rejected": -0.6572116613388062, |
| "step": 281 |
| }, |
| { |
| "dpo_lambda": 0.9705449342727661, |
| "epoch": 0.590421355666056, |
| "grad_norm": 26.60962349078345, |
| "learning_rate": 2.1442129043167873e-07, |
| "logits/chosen": -0.9322452545166016, |
| "logits/rejected": -0.9412853121757507, |
| "logps/chosen": -319.7808532714844, |
| "logps/rejected": -352.3249816894531, |
| "loss": 0.5779, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.3038173317909241, |
| "rewards/margins": 0.4064999520778656, |
| "rewards/rejected": -0.7103172540664673, |
| "step": 282 |
| }, |
| { |
| "dpo_lambda": 0.9704403281211853, |
| "epoch": 0.5925150484166448, |
| "grad_norm": 13.698113538753912, |
| "learning_rate": 2.1261013021512378e-07, |
| "logits/chosen": -1.0408434867858887, |
| "logits/rejected": -1.0634641647338867, |
| "logps/chosen": -324.38421630859375, |
| "logps/rejected": -296.24310302734375, |
| "loss": 0.5603, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.2441764771938324, |
| "rewards/margins": 0.45150530338287354, |
| "rewards/rejected": -0.6956817507743835, |
| "step": 283 |
| }, |
| { |
| "dpo_lambda": 0.9703354239463806, |
| "epoch": 0.5946087411672337, |
| "grad_norm": 35.606967375148706, |
| "learning_rate": 2.1080097510381294e-07, |
| "logits/chosen": -0.9592381119728088, |
| "logits/rejected": -0.9521263241767883, |
| "logps/chosen": -367.49041748046875, |
| "logps/rejected": -358.9259948730469, |
| "loss": 0.572, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.4385915994644165, |
| "rewards/margins": 0.35387519001960754, |
| "rewards/rejected": -0.7924667596817017, |
| "step": 284 |
| }, |
| { |
| "dpo_lambda": 0.9702305793762207, |
| "epoch": 0.5967024339178225, |
| "grad_norm": 35.5832827055, |
| "learning_rate": 2.089939221172446e-07, |
| "logits/chosen": -0.866810142993927, |
| "logits/rejected": -0.9270384311676025, |
| "logps/chosen": -296.6413269042969, |
| "logps/rejected": -340.2668762207031, |
| "loss": 0.534, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -0.17586766183376312, |
| "rewards/margins": 0.633611798286438, |
| "rewards/rejected": -0.8094794154167175, |
| "step": 285 |
| }, |
| { |
| "dpo_lambda": 0.9701257348060608, |
| "epoch": 0.5987961266684114, |
| "grad_norm": 35.734015963321454, |
| "learning_rate": 2.0718906816218595e-07, |
| "logits/chosen": -0.9615485668182373, |
| "logits/rejected": -0.9750257134437561, |
| "logps/chosen": -349.071533203125, |
| "logps/rejected": -384.72247314453125, |
| "loss": 0.5693, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.4143269658088684, |
| "rewards/margins": 0.5034248232841492, |
| "rewards/rejected": -0.9177517890930176, |
| "step": 286 |
| }, |
| { |
| "dpo_lambda": 0.9700208902359009, |
| "epoch": 0.6008898194190002, |
| "grad_norm": 30.680257622648437, |
| "learning_rate": 2.053865100274774e-07, |
| "logits/chosen": -1.0024163722991943, |
| "logits/rejected": -1.057032823562622, |
| "logps/chosen": -356.08160400390625, |
| "logps/rejected": -342.71380615234375, |
| "loss": 0.5694, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.33542853593826294, |
| "rewards/margins": 0.4585578143596649, |
| "rewards/rejected": -0.7939863801002502, |
| "step": 287 |
| }, |
| { |
| "dpo_lambda": 0.9699162244796753, |
| "epoch": 0.6029835121695891, |
| "grad_norm": 23.556702970914774, |
| "learning_rate": 2.035863443788411e-07, |
| "logits/chosen": -0.9257787466049194, |
| "logits/rejected": -0.9421148300170898, |
| "logps/chosen": -327.1122131347656, |
| "logps/rejected": -364.0360412597656, |
| "loss": 0.5885, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.31706058979034424, |
| "rewards/margins": 0.6234011054039001, |
| "rewards/rejected": -0.9404616355895996, |
| "step": 288 |
| }, |
| { |
| "dpo_lambda": 0.9698113799095154, |
| "epoch": 0.6050772049201779, |
| "grad_norm": 28.145132727704436, |
| "learning_rate": 2.0178866775369774e-07, |
| "logits/chosen": -0.9009627103805542, |
| "logits/rejected": -0.9209296107292175, |
| "logps/chosen": -294.1601867675781, |
| "logps/rejected": -330.5748291015625, |
| "loss": 0.5464, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.27767202258110046, |
| "rewards/margins": 0.6135878562927246, |
| "rewards/rejected": -0.8912597894668579, |
| "step": 289 |
| }, |
| { |
| "dpo_lambda": 0.9697064757347107, |
| "epoch": 0.6071708976707668, |
| "grad_norm": 19.389825728653452, |
| "learning_rate": 1.9999357655598891e-07, |
| "logits/chosen": -0.8907105326652527, |
| "logits/rejected": -0.9732505083084106, |
| "logps/chosen": -292.0400085449219, |
| "logps/rejected": -282.02520751953125, |
| "loss": 0.5727, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.28145384788513184, |
| "rewards/margins": 0.5408211350440979, |
| "rewards/rejected": -0.822274923324585, |
| "step": 290 |
| }, |
| { |
| "dpo_lambda": 0.9696016907691956, |
| "epoch": 0.6092645904213556, |
| "grad_norm": 22.906647894771478, |
| "learning_rate": 1.9820116705100775e-07, |
| "logits/chosen": -0.974831759929657, |
| "logits/rejected": -0.9915525317192078, |
| "logps/chosen": -350.82232666015625, |
| "logps/rejected": -394.2654724121094, |
| "loss": 0.6151, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.4027952551841736, |
| "rewards/margins": 0.3092948794364929, |
| "rewards/rejected": -0.7120901346206665, |
| "step": 291 |
| }, |
| { |
| "dpo_lambda": 0.9694967865943909, |
| "epoch": 0.6113582831719445, |
| "grad_norm": 30.636875549929094, |
| "learning_rate": 1.9641153536023642e-07, |
| "logits/chosen": -0.7265560030937195, |
| "logits/rejected": -0.8953325152397156, |
| "logps/chosen": -318.35992431640625, |
| "logps/rejected": -342.0888671875, |
| "loss": 0.5318, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.43135565519332886, |
| "rewards/margins": 0.5195130109786987, |
| "rewards/rejected": -0.9508687257766724, |
| "step": 292 |
| }, |
| { |
| "dpo_lambda": 0.969391942024231, |
| "epoch": 0.6134519759225334, |
| "grad_norm": 22.769319565191545, |
| "learning_rate": 1.9462477745619106e-07, |
| "logits/chosen": -0.9743357300758362, |
| "logits/rejected": -0.9867159128189087, |
| "logps/chosen": -365.81536865234375, |
| "logps/rejected": -400.97064208984375, |
| "loss": 0.6079, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.49305474758148193, |
| "rewards/margins": 0.5272669196128845, |
| "rewards/rejected": -1.0203216075897217, |
| "step": 293 |
| }, |
| { |
| "dpo_lambda": 0.9692872762680054, |
| "epoch": 0.6155456686731222, |
| "grad_norm": 28.096584516595076, |
| "learning_rate": 1.928409891572757e-07, |
| "logits/chosen": -0.9767225980758667, |
| "logits/rejected": -1.018739104270935, |
| "logps/chosen": -290.4533996582031, |
| "logps/rejected": -369.98883056640625, |
| "loss": 0.5211, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -0.24696438014507294, |
| "rewards/margins": 0.6717772483825684, |
| "rewards/rejected": -0.9187415838241577, |
| "step": 294 |
| }, |
| { |
| "dpo_lambda": 0.9691824316978455, |
| "epoch": 0.6176393614237111, |
| "grad_norm": 11.67452650850463, |
| "learning_rate": 1.9106026612264315e-07, |
| "logits/chosen": -0.9304852485656738, |
| "logits/rejected": -0.9730537533760071, |
| "logps/chosen": -334.56304931640625, |
| "logps/rejected": -366.45904541015625, |
| "loss": 0.4976, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.28208717703819275, |
| "rewards/margins": 0.617598295211792, |
| "rewards/rejected": -0.8996855020523071, |
| "step": 295 |
| }, |
| { |
| "dpo_lambda": 0.9690775871276855, |
| "epoch": 0.6197330541742999, |
| "grad_norm": 29.939339000249788, |
| "learning_rate": 1.8928270384706582e-07, |
| "logits/chosen": -0.9597479104995728, |
| "logits/rejected": -1.0559049844741821, |
| "logps/chosen": -361.4211120605469, |
| "logps/rejected": -343.28009033203125, |
| "loss": 0.5373, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.31935498118400574, |
| "rewards/margins": 0.6444027423858643, |
| "rewards/rejected": -0.9637576937675476, |
| "step": 296 |
| }, |
| { |
| "dpo_lambda": 0.9689727425575256, |
| "epoch": 0.6218267469248888, |
| "grad_norm": 19.420346723363878, |
| "learning_rate": 1.875083976558136e-07, |
| "logits/chosen": -0.8957151174545288, |
| "logits/rejected": -0.9098566770553589, |
| "logps/chosen": -330.9611511230469, |
| "logps/rejected": -398.48834228515625, |
| "loss": 0.5834, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.3520338535308838, |
| "rewards/margins": 0.5692517757415771, |
| "rewards/rejected": -0.9212855696678162, |
| "step": 297 |
| }, |
| { |
| "dpo_lambda": 0.968867838382721, |
| "epoch": 0.6239204396754776, |
| "grad_norm": 17.95489479788074, |
| "learning_rate": 1.8573744269954297e-07, |
| "logits/chosen": -0.9044655561447144, |
| "logits/rejected": -0.986747145652771, |
| "logps/chosen": -312.3471984863281, |
| "logps/rejected": -321.05810546875, |
| "loss": 0.5608, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.31143227219581604, |
| "rewards/margins": 0.5159186124801636, |
| "rewards/rejected": -0.8273508548736572, |
| "step": 298 |
| }, |
| { |
| "dpo_lambda": 0.9687632322311401, |
| "epoch": 0.6260141324260665, |
| "grad_norm": 15.662522135428798, |
| "learning_rate": 1.839699339491937e-07, |
| "logits/chosen": -0.9157642126083374, |
| "logits/rejected": -0.9654196500778198, |
| "logps/chosen": -341.7589416503906, |
| "logps/rejected": -348.1306457519531, |
| "loss": 0.5487, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.28629571199417114, |
| "rewards/margins": 0.6190310716629028, |
| "rewards/rejected": -0.905326783657074, |
| "step": 299 |
| }, |
| { |
| "dpo_lambda": 0.9686583280563354, |
| "epoch": 0.6281078251766553, |
| "grad_norm": 23.191123388712988, |
| "learning_rate": 1.8220596619089573e-07, |
| "logits/chosen": -0.9885008931159973, |
| "logits/rejected": -0.9450974464416504, |
| "logps/chosen": -309.62091064453125, |
| "logps/rejected": -363.9620361328125, |
| "loss": 0.5598, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.4183036684989929, |
| "rewards/margins": 0.5146893262863159, |
| "rewards/rejected": -0.9329929947853088, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.6281078251766553, |
| "eval_dpo_lambda": 0.9685534834861755, |
| "eval_logits/chosen": -0.9607605934143066, |
| "eval_logits/rejected": -1.0002164840698242, |
| "eval_logps/chosen": -343.1056823730469, |
| "eval_logps/rejected": -361.8922424316406, |
| "eval_loss": 0.5694788694381714, |
| "eval_rewards/accuracies": 0.722000002861023, |
| "eval_rewards/chosen": -0.4245879352092743, |
| "eval_rewards/margins": 0.4840264618396759, |
| "eval_rewards/rejected": -0.9086143374443054, |
| "eval_runtime": 561.1076, |
| "eval_samples_per_second": 3.564, |
| "eval_steps_per_second": 0.891, |
| "step": 300 |
| }, |
| { |
| "dpo_lambda": 0.9685534834861755, |
| "epoch": 0.6302015179272442, |
| "grad_norm": 71.58397918802766, |
| "learning_rate": 1.8044563402088682e-07, |
| "logits/chosen": -0.9689053893089294, |
| "logits/rejected": -1.0844143629074097, |
| "logps/chosen": -326.18499755859375, |
| "logps/rejected": -329.495849609375, |
| "loss": 0.5791, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.5225777626037598, |
| "rewards/margins": 0.3462575674057007, |
| "rewards/rejected": -0.86883544921875, |
| "step": 301 |
| }, |
| { |
| "dpo_lambda": 0.9684486389160156, |
| "epoch": 0.632295210677833, |
| "grad_norm": 26.151243700328262, |
| "learning_rate": 1.7868903184043885e-07, |
| "logits/chosen": -0.8586608171463013, |
| "logits/rejected": -0.9734141230583191, |
| "logps/chosen": -307.2423400878906, |
| "logps/rejected": -328.2541809082031, |
| "loss": 0.5424, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.3227950930595398, |
| "rewards/margins": 0.5719585418701172, |
| "rewards/rejected": -0.8947535753250122, |
| "step": 302 |
| }, |
| { |
| "dpo_lambda": 0.9683437943458557, |
| "epoch": 0.6343889034284219, |
| "grad_norm": 18.993097696585455, |
| "learning_rate": 1.7693625385079574e-07, |
| "logits/chosen": -0.9815896153450012, |
| "logits/rejected": -1.009246826171875, |
| "logps/chosen": -317.59832763671875, |
| "logps/rejected": -327.9718322753906, |
| "loss": 0.5344, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.23856814205646515, |
| "rewards/margins": 0.5028955936431885, |
| "rewards/rejected": -0.7414637207984924, |
| "step": 303 |
| }, |
| { |
| "dpo_lambda": 0.9682389497756958, |
| "epoch": 0.6364825961790107, |
| "grad_norm": 53.94254872286131, |
| "learning_rate": 1.7518739404812155e-07, |
| "logits/chosen": -0.9886007308959961, |
| "logits/rejected": -1.0203882455825806, |
| "logps/chosen": -387.0471496582031, |
| "logps/rejected": -371.6809997558594, |
| "loss": 0.5769, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.5815944671630859, |
| "rewards/margins": 0.38539034128189087, |
| "rewards/rejected": -0.966984748840332, |
| "step": 304 |
| }, |
| { |
| "dpo_lambda": 0.9681340456008911, |
| "epoch": 0.6385762889295996, |
| "grad_norm": 26.17411762649003, |
| "learning_rate": 1.7344254621846017e-07, |
| "logits/chosen": -1.0002169609069824, |
| "logits/rejected": -1.000640869140625, |
| "logps/chosen": -312.68365478515625, |
| "logps/rejected": -287.2958068847656, |
| "loss": 0.5631, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.35965269804000854, |
| "rewards/margins": 0.4842557907104492, |
| "rewards/rejected": -0.8439084887504578, |
| "step": 305 |
| }, |
| { |
| "dpo_lambda": 0.9680293798446655, |
| "epoch": 0.6406699816801884, |
| "grad_norm": 25.76044753092561, |
| "learning_rate": 1.717018039327053e-07, |
| "logits/chosen": -0.8985933065414429, |
| "logits/rejected": -0.9996975064277649, |
| "logps/chosen": -389.3902282714844, |
| "logps/rejected": -382.3996887207031, |
| "loss": 0.5286, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.32364633679389954, |
| "rewards/margins": 0.6448994278907776, |
| "rewards/rejected": -0.9685457944869995, |
| "step": 306 |
| }, |
| { |
| "dpo_lambda": 0.9679245352745056, |
| "epoch": 0.6427636744307773, |
| "grad_norm": 21.07447363401618, |
| "learning_rate": 1.699652605415828e-07, |
| "logits/chosen": -0.9501347541809082, |
| "logits/rejected": -1.0148643255233765, |
| "logps/chosen": -408.3487243652344, |
| "logps/rejected": -361.47979736328125, |
| "loss": 0.5062, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.41872522234916687, |
| "rewards/margins": 0.4738251268863678, |
| "rewards/rejected": -0.8925502896308899, |
| "step": 307 |
| }, |
| { |
| "dpo_lambda": 0.9678196907043457, |
| "epoch": 0.6448573671813661, |
| "grad_norm": 37.352068506484386, |
| "learning_rate": 1.6823300917064458e-07, |
| "logits/chosen": -0.9194483160972595, |
| "logits/rejected": -1.0151333808898926, |
| "logps/chosen": -348.45068359375, |
| "logps/rejected": -390.6669921875, |
| "loss": 0.5342, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.37650251388549805, |
| "rewards/margins": 0.5100971460342407, |
| "rewards/rejected": -0.886599600315094, |
| "step": 308 |
| }, |
| { |
| "dpo_lambda": 0.9677148461341858, |
| "epoch": 0.646951059931955, |
| "grad_norm": 65.9272239651892, |
| "learning_rate": 1.6650514271527465e-07, |
| "logits/chosen": -0.8971225619316101, |
| "logits/rejected": -0.9499090909957886, |
| "logps/chosen": -341.97503662109375, |
| "logps/rejected": -398.912353515625, |
| "loss": 0.558, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.261827677488327, |
| "rewards/margins": 0.5827478766441345, |
| "rewards/rejected": -0.8445755243301392, |
| "step": 309 |
| }, |
| { |
| "dpo_lambda": 0.9676100015640259, |
| "epoch": 0.6490447526825438, |
| "grad_norm": 20.04658084797354, |
| "learning_rate": 1.647817538357072e-07, |
| "logits/chosen": -0.7898070812225342, |
| "logits/rejected": -0.8542658090591431, |
| "logps/chosen": -318.4727783203125, |
| "logps/rejected": -329.48333740234375, |
| "loss": 0.5176, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.29877397418022156, |
| "rewards/margins": 0.5912772417068481, |
| "rewards/rejected": -0.8900513052940369, |
| "step": 310 |
| }, |
| { |
| "dpo_lambda": 0.9675053358078003, |
| "epoch": 0.6511384454331327, |
| "grad_norm": 26.373458404599162, |
| "learning_rate": 1.6306293495205755e-07, |
| "logits/chosen": -0.9485715627670288, |
| "logits/rejected": -0.9856311082839966, |
| "logps/chosen": -303.1679382324219, |
| "logps/rejected": -325.40655517578125, |
| "loss": 0.538, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.3055274486541748, |
| "rewards/margins": 0.5957139134407043, |
| "rewards/rejected": -0.9012414216995239, |
| "step": 311 |
| }, |
| { |
| "dpo_lambda": 0.9674004912376404, |
| "epoch": 0.6532321381837215, |
| "grad_norm": 15.730910702810599, |
| "learning_rate": 1.6134877823936607e-07, |
| "logits/chosen": -1.0132697820663452, |
| "logits/rejected": -1.0205020904541016, |
| "logps/chosen": -329.7890319824219, |
| "logps/rejected": -403.00732421875, |
| "loss": 0.5396, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.3859696388244629, |
| "rewards/margins": 0.5894760489463806, |
| "rewards/rejected": -0.9754457473754883, |
| "step": 312 |
| }, |
| { |
| "dpo_lambda": 0.9672955870628357, |
| "epoch": 0.6553258309343104, |
| "grad_norm": 28.053518321597984, |
| "learning_rate": 1.5963937562265522e-07, |
| "logits/chosen": -0.9850528240203857, |
| "logits/rejected": -1.0680859088897705, |
| "logps/chosen": -329.48236083984375, |
| "logps/rejected": -358.2930908203125, |
| "loss": 0.5493, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.25245147943496704, |
| "rewards/margins": 0.5047082901000977, |
| "rewards/rejected": -0.7571598291397095, |
| "step": 313 |
| }, |
| { |
| "dpo_lambda": 0.9671907424926758, |
| "epoch": 0.6574195236848992, |
| "grad_norm": 44.39330956194096, |
| "learning_rate": 1.5793481877199943e-07, |
| "logits/chosen": -1.0563923120498657, |
| "logits/rejected": -1.0740464925765991, |
| "logps/chosen": -327.7634582519531, |
| "logps/rejected": -326.1741943359375, |
| "loss": 0.5415, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.1798219382762909, |
| "rewards/margins": 0.5696390867233276, |
| "rewards/rejected": -0.7494610548019409, |
| "step": 314 |
| }, |
| { |
| "dpo_lambda": 0.9670858979225159, |
| "epoch": 0.6595132164354881, |
| "grad_norm": 52.35196491982725, |
| "learning_rate": 1.562351990976095e-07, |
| "logits/chosen": -0.8370614051818848, |
| "logits/rejected": -0.9014161825180054, |
| "logps/chosen": -346.0189208984375, |
| "logps/rejected": -366.05963134765625, |
| "loss": 0.5071, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.19401825964450836, |
| "rewards/margins": 0.7404112219810486, |
| "rewards/rejected": -0.934429407119751, |
| "step": 315 |
| }, |
| { |
| "dpo_lambda": 0.966981053352356, |
| "epoch": 0.6616069091860769, |
| "grad_norm": 20.25931696368988, |
| "learning_rate": 1.5454060774493065e-07, |
| "logits/chosen": -0.9522421360015869, |
| "logits/rejected": -0.9345359802246094, |
| "logps/chosen": -300.9903564453125, |
| "logps/rejected": -332.50396728515625, |
| "loss": 0.5295, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.33971095085144043, |
| "rewards/margins": 0.5237520337104797, |
| "rewards/rejected": -0.8634629249572754, |
| "step": 316 |
| }, |
| { |
| "dpo_lambda": 0.9668763875961304, |
| "epoch": 0.6637006019366658, |
| "grad_norm": 28.05666808675557, |
| "learning_rate": 1.5285113558975427e-07, |
| "logits/chosen": -0.9864431619644165, |
| "logits/rejected": -0.94712233543396, |
| "logps/chosen": -314.5718688964844, |
| "logps/rejected": -369.55609130859375, |
| "loss": 0.5285, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.3466428816318512, |
| "rewards/margins": 0.5084124207496643, |
| "rewards/rejected": -0.8550552725791931, |
| "step": 317 |
| }, |
| { |
| "dpo_lambda": 0.9667715430259705, |
| "epoch": 0.6657942946872546, |
| "grad_norm": 28.907111649527103, |
| "learning_rate": 1.5116687323334464e-07, |
| "logits/chosen": -1.031725525856018, |
| "logits/rejected": -1.0288310050964355, |
| "logps/chosen": -351.31671142578125, |
| "logps/rejected": -391.8863220214844, |
| "loss": 0.5332, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.29358726739883423, |
| "rewards/margins": 0.6588489413261414, |
| "rewards/rejected": -0.9524362683296204, |
| "step": 318 |
| }, |
| { |
| "dpo_lambda": 0.9666666388511658, |
| "epoch": 0.6678879874378435, |
| "grad_norm": 13.586327459344083, |
| "learning_rate": 1.4948791099758052e-07, |
| "logits/chosen": -0.940189003944397, |
| "logits/rejected": -0.968927264213562, |
| "logps/chosen": -283.3269958496094, |
| "logps/rejected": -314.5475769042969, |
| "loss": 0.5594, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.2659313678741455, |
| "rewards/margins": 0.5632839202880859, |
| "rewards/rejected": -0.829215407371521, |
| "step": 319 |
| }, |
| { |
| "dpo_lambda": 0.9665618538856506, |
| "epoch": 0.6699816801884323, |
| "grad_norm": 17.287945972540122, |
| "learning_rate": 1.478143389201113e-07, |
| "logits/chosen": -0.9991787075996399, |
| "logits/rejected": -0.9350276589393616, |
| "logps/chosen": -420.7625427246094, |
| "logps/rejected": -389.25408935546875, |
| "loss": 0.5606, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.39572352170944214, |
| "rewards/margins": 0.4197995960712433, |
| "rewards/rejected": -0.8155230283737183, |
| "step": 320 |
| }, |
| { |
| "dpo_lambda": 0.966456949710846, |
| "epoch": 0.6720753729390212, |
| "grad_norm": 22.059789067729692, |
| "learning_rate": 1.461462467495284e-07, |
| "logits/chosen": -0.8892531991004944, |
| "logits/rejected": -0.9207965135574341, |
| "logps/chosen": -371.59234619140625, |
| "logps/rejected": -356.5474548339844, |
| "loss": 0.6141, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.3681826591491699, |
| "rewards/margins": 0.5087204575538635, |
| "rewards/rejected": -0.8769031763076782, |
| "step": 321 |
| }, |
| { |
| "dpo_lambda": 0.9663523435592651, |
| "epoch": 0.6741690656896101, |
| "grad_norm": 17.733315729809913, |
| "learning_rate": 1.4448372394055246e-07, |
| "logits/chosen": -0.9744499325752258, |
| "logits/rejected": -0.9986258745193481, |
| "logps/chosen": -321.5147399902344, |
| "logps/rejected": -330.0378112792969, |
| "loss": 0.517, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.30931758880615234, |
| "rewards/margins": 0.5616300106048584, |
| "rewards/rejected": -0.870947539806366, |
| "step": 322 |
| }, |
| { |
| "dpo_lambda": 0.9662474393844604, |
| "epoch": 0.6762627584401989, |
| "grad_norm": 23.90951181882235, |
| "learning_rate": 1.428268596492364e-07, |
| "logits/chosen": -0.8879707455635071, |
| "logits/rejected": -0.8915808796882629, |
| "logps/chosen": -326.7454528808594, |
| "logps/rejected": -378.8501892089844, |
| "loss": 0.5528, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.325499564409256, |
| "rewards/margins": 0.5490182042121887, |
| "rewards/rejected": -0.8745177984237671, |
| "step": 323 |
| }, |
| { |
| "dpo_lambda": 0.9661425948143005, |
| "epoch": 0.6783564511907878, |
| "grad_norm": 16.39553700201974, |
| "learning_rate": 1.4117574272818386e-07, |
| "logits/chosen": -0.9681739211082458, |
| "logits/rejected": -1.015541911125183, |
| "logps/chosen": -317.4237976074219, |
| "logps/rejected": -372.75439453125, |
| "loss": 0.5674, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.33222252130508423, |
| "rewards/margins": 0.5510666966438293, |
| "rewards/rejected": -0.8832892179489136, |
| "step": 324 |
| }, |
| { |
| "dpo_lambda": 0.9660377502441406, |
| "epoch": 0.6804501439413766, |
| "grad_norm": 18.791465531764167, |
| "learning_rate": 1.3953046172178413e-07, |
| "logits/chosen": -0.9151560664176941, |
| "logits/rejected": -0.9827873706817627, |
| "logps/chosen": -323.977783203125, |
| "logps/rejected": -314.0232238769531, |
| "loss": 0.5845, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.4418525993824005, |
| "rewards/margins": 0.37086665630340576, |
| "rewards/rejected": -0.8127192258834839, |
| "step": 325 |
| }, |
| { |
| "dpo_lambda": 0.9659329056739807, |
| "epoch": 0.6825438366919655, |
| "grad_norm": 27.99201137301602, |
| "learning_rate": 1.3789110486146468e-07, |
| "logits/chosen": -1.0051405429840088, |
| "logits/rejected": -1.0525906085968018, |
| "logps/chosen": -324.9797668457031, |
| "logps/rejected": -370.70794677734375, |
| "loss": 0.5747, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.27201488614082336, |
| "rewards/margins": 0.48317158222198486, |
| "rewards/rejected": -0.7551864385604858, |
| "step": 326 |
| }, |
| { |
| "dpo_lambda": 0.965828001499176, |
| "epoch": 0.6846375294425543, |
| "grad_norm": 20.528135742260847, |
| "learning_rate": 1.362577600609588e-07, |
| "logits/chosen": -1.0105348825454712, |
| "logits/rejected": -0.9638998508453369, |
| "logps/chosen": -318.3458251953125, |
| "logps/rejected": -370.9636535644531, |
| "loss": 0.555, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.293737530708313, |
| "rewards/margins": 0.45407989621162415, |
| "rewards/rejected": -0.74781733751297, |
| "step": 327 |
| }, |
| { |
| "dpo_lambda": 0.9657231569290161, |
| "epoch": 0.6867312221931432, |
| "grad_norm": 40.497052517569266, |
| "learning_rate": 1.3463051491159093e-07, |
| "logits/chosen": -0.9311863780021667, |
| "logits/rejected": -0.9145044684410095, |
| "logps/chosen": -306.9552001953125, |
| "logps/rejected": -379.8552551269531, |
| "loss": 0.5613, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.3483826816082001, |
| "rewards/margins": 0.598532497882843, |
| "rewards/rejected": -0.9469150900840759, |
| "step": 328 |
| }, |
| { |
| "dpo_lambda": 0.9656184911727905, |
| "epoch": 0.688824914943732, |
| "grad_norm": 16.832445288233277, |
| "learning_rate": 1.3300945667758012e-07, |
| "logits/chosen": -0.8724278807640076, |
| "logits/rejected": -0.9922559261322021, |
| "logps/chosen": -349.7203369140625, |
| "logps/rejected": -389.9562683105469, |
| "loss": 0.5049, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.22436396777629852, |
| "rewards/margins": 0.7489404678344727, |
| "rewards/rejected": -0.9733043909072876, |
| "step": 329 |
| }, |
| { |
| "dpo_lambda": 0.9655136466026306, |
| "epoch": 0.6909186076943209, |
| "grad_norm": 26.95436484029208, |
| "learning_rate": 1.3139467229135998e-07, |
| "logits/chosen": -0.9775909185409546, |
| "logits/rejected": -0.9572923183441162, |
| "logps/chosen": -296.727783203125, |
| "logps/rejected": -318.11578369140625, |
| "loss": 0.5528, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.32260772585868835, |
| "rewards/margins": 0.43575817346572876, |
| "rewards/rejected": -0.7583658695220947, |
| "step": 330 |
| }, |
| { |
| "dpo_lambda": 0.9654088020324707, |
| "epoch": 0.6930123004449097, |
| "grad_norm": 21.94704622393633, |
| "learning_rate": 1.2978624834891626e-07, |
| "logits/chosen": -0.856591522693634, |
| "logits/rejected": -0.925330638885498, |
| "logps/chosen": -363.8154602050781, |
| "logps/rejected": -351.26409912109375, |
| "loss": 0.4924, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.2571355402469635, |
| "rewards/margins": 0.6332350373268127, |
| "rewards/rejected": -0.8903706669807434, |
| "step": 331 |
| }, |
| { |
| "dpo_lambda": 0.9653039574623108, |
| "epoch": 0.6951059931954986, |
| "grad_norm": 178.8370298654354, |
| "learning_rate": 1.281842711051438e-07, |
| "logits/chosen": -1.0046966075897217, |
| "logits/rejected": -0.9471710920333862, |
| "logps/chosen": -407.97955322265625, |
| "logps/rejected": -443.640869140625, |
| "loss": 0.5206, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -0.3523060083389282, |
| "rewards/margins": 0.6791791915893555, |
| "rewards/rejected": -1.0314850807189941, |
| "step": 332 |
| }, |
| { |
| "dpo_lambda": 0.9651991128921509, |
| "epoch": 0.6971996859460874, |
| "grad_norm": 34.231688437828396, |
| "learning_rate": 1.2658882646922033e-07, |
| "logits/chosen": -0.8789958357810974, |
| "logits/rejected": -0.9038445949554443, |
| "logps/chosen": -354.1581726074219, |
| "logps/rejected": -366.3446044921875, |
| "loss": 0.5678, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.36087289452552795, |
| "rewards/margins": 0.5995725393295288, |
| "rewards/rejected": -0.9604454040527344, |
| "step": 333 |
| }, |
| { |
| "dpo_lambda": 0.9650944471359253, |
| "epoch": 0.6992933786966763, |
| "grad_norm": 18.04385028677772, |
| "learning_rate": 1.2500000000000005e-07, |
| "logits/chosen": -0.9685949683189392, |
| "logits/rejected": -0.9957066774368286, |
| "logps/chosen": -335.9698181152344, |
| "logps/rejected": -395.140869140625, |
| "loss": 0.5561, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.30749112367630005, |
| "rewards/margins": 0.4485364258289337, |
| "rewards/rejected": -0.7560275793075562, |
| "step": 334 |
| }, |
| { |
| "dpo_lambda": 0.9649895429611206, |
| "epoch": 0.7013870714472651, |
| "grad_norm": 21.053487705520137, |
| "learning_rate": 1.2341787690142435e-07, |
| "logits/chosen": -0.9309602379798889, |
| "logits/rejected": -0.963007926940918, |
| "logps/chosen": -331.0724792480469, |
| "logps/rejected": -347.17486572265625, |
| "loss": 0.5789, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.40828946232795715, |
| "rewards/margins": 0.4629323184490204, |
| "rewards/rejected": -0.8712217807769775, |
| "step": 335 |
| }, |
| { |
| "dpo_lambda": 0.9648846983909607, |
| "epoch": 0.703480764197854, |
| "grad_norm": 20.90654959482647, |
| "learning_rate": 1.2184254201795363e-07, |
| "logits/chosen": -0.935300350189209, |
| "logits/rejected": -1.0370335578918457, |
| "logps/chosen": -361.41436767578125, |
| "logps/rejected": -372.24920654296875, |
| "loss": 0.5524, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.3370175063610077, |
| "rewards/margins": 0.41511785984039307, |
| "rewards/rejected": -0.7521353960037231, |
| "step": 336 |
| }, |
| { |
| "dpo_lambda": 0.9647798538208008, |
| "epoch": 0.7055744569484428, |
| "grad_norm": 25.47986436291456, |
| "learning_rate": 1.202740798300168e-07, |
| "logits/chosen": -1.0110206604003906, |
| "logits/rejected": -0.9602785110473633, |
| "logps/chosen": -369.0711669921875, |
| "logps/rejected": -426.0411376953125, |
| "loss": 0.5572, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.5039616823196411, |
| "rewards/margins": 0.48775193095207214, |
| "rewards/rejected": -0.9917135834693909, |
| "step": 337 |
| }, |
| { |
| "dpo_lambda": 0.9646750092506409, |
| "epoch": 0.7076681496990317, |
| "grad_norm": 12.80550446526403, |
| "learning_rate": 1.1871257444948096e-07, |
| "logits/chosen": -0.959979772567749, |
| "logits/rejected": -1.0622349977493286, |
| "logps/chosen": -339.4168395996094, |
| "logps/rejected": -341.2720947265625, |
| "loss": 0.5448, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.4172990918159485, |
| "rewards/margins": 0.5843453407287598, |
| "rewards/rejected": -1.0016443729400635, |
| "step": 338 |
| }, |
| { |
| "dpo_lambda": 0.964570164680481, |
| "epoch": 0.7097618424496205, |
| "grad_norm": 24.972770330893827, |
| "learning_rate": 1.1715810961514072e-07, |
| "logits/chosen": -0.8925620317459106, |
| "logits/rejected": -0.9319013357162476, |
| "logps/chosen": -293.1116638183594, |
| "logps/rejected": -370.85284423828125, |
| "loss": 0.5661, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.2533493638038635, |
| "rewards/margins": 0.6773079037666321, |
| "rewards/rejected": -0.9306572079658508, |
| "step": 339 |
| }, |
| { |
| "dpo_lambda": 0.9644652605056763, |
| "epoch": 0.7118555352002094, |
| "grad_norm": 22.919753073070936, |
| "learning_rate": 1.1561076868822755e-07, |
| "logits/chosen": -0.9318410158157349, |
| "logits/rejected": -0.9421119689941406, |
| "logps/chosen": -321.55072021484375, |
| "logps/rejected": -356.93603515625, |
| "loss": 0.595, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.32962632179260254, |
| "rewards/margins": 0.37742599844932556, |
| "rewards/rejected": -0.7070522308349609, |
| "step": 340 |
| }, |
| { |
| "dpo_lambda": 0.9643606543540955, |
| "epoch": 0.7139492279507982, |
| "grad_norm": 29.275505863100147, |
| "learning_rate": 1.1407063464793965e-07, |
| "logits/chosen": -1.0172332525253296, |
| "logits/rejected": -1.0134183168411255, |
| "logps/chosen": -278.432861328125, |
| "logps/rejected": -322.0915222167969, |
| "loss": 0.5227, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.3785405158996582, |
| "rewards/margins": 0.5554640293121338, |
| "rewards/rejected": -0.9340046048164368, |
| "step": 341 |
| }, |
| { |
| "dpo_lambda": 0.9642557501792908, |
| "epoch": 0.7160429207013871, |
| "grad_norm": 35.203314569620424, |
| "learning_rate": 1.125377900869913e-07, |
| "logits/chosen": -0.9718924164772034, |
| "logits/rejected": -0.9619014859199524, |
| "logps/chosen": -321.0333251953125, |
| "logps/rejected": -392.4592590332031, |
| "loss": 0.5256, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.2639404237270355, |
| "rewards/margins": 0.6683336496353149, |
| "rewards/rejected": -0.9322740435600281, |
| "step": 342 |
| }, |
| { |
| "dpo_lambda": 0.9641509056091309, |
| "epoch": 0.7181366134519759, |
| "grad_norm": 55.55269901625133, |
| "learning_rate": 1.110123172071844e-07, |
| "logits/chosen": -0.914270281791687, |
| "logits/rejected": -0.860942006111145, |
| "logps/chosen": -304.1083068847656, |
| "logps/rejected": -381.48651123046875, |
| "loss": 0.5444, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.2739492952823639, |
| "rewards/margins": 0.5674920678138733, |
| "rewards/rejected": -0.8414413332939148, |
| "step": 343 |
| }, |
| { |
| "dpo_lambda": 0.964046061038971, |
| "epoch": 0.7202303062025648, |
| "grad_norm": 17.912467497986132, |
| "learning_rate": 1.09494297815e-07, |
| "logits/chosen": -0.9377856850624084, |
| "logits/rejected": -1.0394203662872314, |
| "logps/chosen": -314.60284423828125, |
| "logps/rejected": -331.8394775390625, |
| "loss": 0.5344, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.25695669651031494, |
| "rewards/margins": 0.441311776638031, |
| "rewards/rejected": -0.6982684135437012, |
| "step": 344 |
| }, |
| { |
| "dpo_lambda": 0.963941216468811, |
| "epoch": 0.7223239989531536, |
| "grad_norm": 14.243821913967022, |
| "learning_rate": 1.0798381331721107e-07, |
| "logits/chosen": -0.9512354731559753, |
| "logits/rejected": -0.9822427034378052, |
| "logps/chosen": -287.7138671875, |
| "logps/rejected": -316.0299072265625, |
| "loss": 0.5271, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.3327217400074005, |
| "rewards/margins": 0.6620177626609802, |
| "rewards/rejected": -0.9947394728660583, |
| "step": 345 |
| }, |
| { |
| "dpo_lambda": 0.9638365507125854, |
| "epoch": 0.7244176917037425, |
| "grad_norm": 15.398034329860014, |
| "learning_rate": 1.0648094471651722e-07, |
| "logits/chosen": -0.9969007968902588, |
| "logits/rejected": -0.9967204928398132, |
| "logps/chosen": -319.8764953613281, |
| "logps/rejected": -340.1079406738281, |
| "loss": 0.5834, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.35108503699302673, |
| "rewards/margins": 0.5176693797111511, |
| "rewards/rejected": -0.8687544465065002, |
| "step": 346 |
| }, |
| { |
| "dpo_lambda": 0.9637317061424255, |
| "epoch": 0.7265113844543313, |
| "grad_norm": 24.431486213535006, |
| "learning_rate": 1.0498577260720048e-07, |
| "logits/chosen": -0.8135185241699219, |
| "logits/rejected": -0.9067487120628357, |
| "logps/chosen": -316.9966125488281, |
| "logps/rejected": -341.70318603515625, |
| "loss": 0.5464, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.41448765993118286, |
| "rewards/margins": 0.4467889368534088, |
| "rewards/rejected": -0.8612766265869141, |
| "step": 347 |
| }, |
| { |
| "dpo_lambda": 0.9636268019676208, |
| "epoch": 0.7286050772049202, |
| "grad_norm": 28.722137917702902, |
| "learning_rate": 1.0349837717080347e-07, |
| "logits/chosen": -0.9912142753601074, |
| "logits/rejected": -1.0412789583206177, |
| "logps/chosen": -357.5257873535156, |
| "logps/rejected": -371.5506591796875, |
| "loss": 0.5016, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.33226823806762695, |
| "rewards/margins": 0.5479937791824341, |
| "rewards/rejected": -0.880262017250061, |
| "step": 348 |
| }, |
| { |
| "dpo_lambda": 0.9635220170021057, |
| "epoch": 0.730698769955509, |
| "grad_norm": 33.358713923011784, |
| "learning_rate": 1.0201883817182949e-07, |
| "logits/chosen": -1.0769925117492676, |
| "logits/rejected": -1.0619198083877563, |
| "logps/chosen": -268.9877014160156, |
| "logps/rejected": -304.9781799316406, |
| "loss": 0.5214, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.3280823230743408, |
| "rewards/margins": 0.5363447666168213, |
| "rewards/rejected": -0.8644271492958069, |
| "step": 349 |
| }, |
| { |
| "dpo_lambda": 0.963417112827301, |
| "epoch": 0.7327924627060979, |
| "grad_norm": 16.957391268850262, |
| "learning_rate": 1.0054723495346482e-07, |
| "logits/chosen": -0.9406857490539551, |
| "logits/rejected": -0.9522515535354614, |
| "logps/chosen": -370.5696105957031, |
| "logps/rejected": -418.59381103515625, |
| "loss": 0.566, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.3182927072048187, |
| "rewards/margins": 0.6044560670852661, |
| "rewards/rejected": -0.9227486848831177, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.7327924627060979, |
| "eval_dpo_lambda": 0.9633123278617859, |
| "eval_logits/chosen": -0.9592387676239014, |
| "eval_logits/rejected": -0.9957619905471802, |
| "eval_logps/chosen": -335.34930419921875, |
| "eval_logps/rejected": -355.07366943359375, |
| "eval_loss": 0.5613225698471069, |
| "eval_rewards/accuracies": 0.7260000109672546, |
| "eval_rewards/chosen": -0.34702369570732117, |
| "eval_rewards/margins": 0.49340489506721497, |
| "eval_rewards/rejected": -0.8404285311698914, |
| "eval_runtime": 561.9253, |
| "eval_samples_per_second": 3.559, |
| "eval_steps_per_second": 0.89, |
| "step": 350 |
| }, |
| { |
| "dpo_lambda": 0.9633122682571411, |
| "epoch": 0.7348861554566868, |
| "grad_norm": 24.71599545821725, |
| "learning_rate": 9.908364643332398e-08, |
| "logits/chosen": -0.9522049427032471, |
| "logits/rejected": -0.9981221556663513, |
| "logps/chosen": -317.208251953125, |
| "logps/rejected": -374.0013427734375, |
| "loss": 0.5682, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.34796202182769775, |
| "rewards/margins": 0.4574374258518219, |
| "rewards/rejected": -0.8053994178771973, |
| "step": 351 |
| }, |
| { |
| "dpo_lambda": 0.9632076025009155, |
| "epoch": 0.7369798482072756, |
| "grad_norm": 15.696672979497066, |
| "learning_rate": 9.76281510992176e-08, |
| "logits/chosen": -0.9699813723564148, |
| "logits/rejected": -1.0038105249404907, |
| "logps/chosen": -323.82733154296875, |
| "logps/rejected": -332.93475341796875, |
| "loss": 0.563, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.3269641697406769, |
| "rewards/margins": 0.5215096473693848, |
| "rewards/rejected": -0.8484737873077393, |
| "step": 352 |
| }, |
| { |
| "dpo_lambda": 0.9631027579307556, |
| "epoch": 0.7390735409578645, |
| "grad_norm": 38.2658159785458, |
| "learning_rate": 9.618082700494318e-08, |
| "logits/chosen": -0.8864728212356567, |
| "logits/rejected": -1.0044827461242676, |
| "logps/chosen": -291.521728515625, |
| "logps/rejected": -291.72802734375, |
| "loss": 0.62, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.41974952816963196, |
| "rewards/margins": 0.38726168870925903, |
| "rewards/rejected": -0.8070113062858582, |
| "step": 353 |
| }, |
| { |
| "dpo_lambda": 0.9629979133605957, |
| "epoch": 0.7411672337084533, |
| "grad_norm": 16.091443237501974, |
| "learning_rate": 9.474175176609956e-08, |
| "logits/chosen": -0.972978413105011, |
| "logits/rejected": -0.9209941625595093, |
| "logps/chosen": -299.625244140625, |
| "logps/rejected": -310.5818786621094, |
| "loss": 0.5659, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.3460564911365509, |
| "rewards/margins": 0.40129145979881287, |
| "rewards/rejected": -0.7473480105400085, |
| "step": 354 |
| }, |
| { |
| "dpo_lambda": 0.9628930687904358, |
| "epoch": 0.7432609264590422, |
| "grad_norm": 28.036122454792796, |
| "learning_rate": 9.331100255592436e-08, |
| "logits/chosen": -0.9111910462379456, |
| "logits/rejected": -0.9185968041419983, |
| "logps/chosen": -379.6253356933594, |
| "logps/rejected": -339.806884765625, |
| "loss": 0.6143, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.45352065563201904, |
| "rewards/margins": 0.28414058685302734, |
| "rewards/rejected": -0.7376612424850464, |
| "step": 355 |
| }, |
| { |
| "dpo_lambda": 0.9627881646156311, |
| "epoch": 0.745354619209631, |
| "grad_norm": 15.812271707140557, |
| "learning_rate": 9.18886561011557e-08, |
| "logits/chosen": -0.8909232020378113, |
| "logits/rejected": -0.9465504884719849, |
| "logps/chosen": -372.94317626953125, |
| "logps/rejected": -410.23406982421875, |
| "loss": 0.518, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.2967568039894104, |
| "rewards/margins": 0.6505753397941589, |
| "rewards/rejected": -0.9473320841789246, |
| "step": 356 |
| }, |
| { |
| "dpo_lambda": 0.9626835584640503, |
| "epoch": 0.7474483119602199, |
| "grad_norm": 17.91342430337661, |
| "learning_rate": 9.047478867791731e-08, |
| "logits/chosen": -0.9878717064857483, |
| "logits/rejected": -0.9517794847488403, |
| "logps/chosen": -346.8152770996094, |
| "logps/rejected": -356.7705993652344, |
| "loss": 0.5188, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.24196535348892212, |
| "rewards/margins": 0.5725304484367371, |
| "rewards/rejected": -0.814495861530304, |
| "step": 357 |
| }, |
| { |
| "dpo_lambda": 0.9625786542892456, |
| "epoch": 0.7495420047108087, |
| "grad_norm": 22.723355184888202, |
| "learning_rate": 8.906947610762825e-08, |
| "logits/chosen": -0.9669464826583862, |
| "logits/rejected": -0.9961766004562378, |
| "logps/chosen": -387.2091064453125, |
| "logps/rejected": -412.39404296875, |
| "loss": 0.5696, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.2763371467590332, |
| "rewards/margins": 0.541536271572113, |
| "rewards/rejected": -0.817873477935791, |
| "step": 358 |
| }, |
| { |
| "dpo_lambda": 0.9624738097190857, |
| "epoch": 0.7516356974613976, |
| "grad_norm": 17.29151886084971, |
| "learning_rate": 8.76727937529367e-08, |
| "logits/chosen": -1.1109174489974976, |
| "logits/rejected": -1.0934998989105225, |
| "logps/chosen": -317.03350830078125, |
| "logps/rejected": -388.8250732421875, |
| "loss": 0.6099, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.30949756503105164, |
| "rewards/margins": 0.31883934140205383, |
| "rewards/rejected": -0.6283369064331055, |
| "step": 359 |
| }, |
| { |
| "dpo_lambda": 0.9623689651489258, |
| "epoch": 0.7537293902119864, |
| "grad_norm": 14.070230696764554, |
| "learning_rate": 8.628481651367875e-08, |
| "logits/chosen": -1.0067811012268066, |
| "logits/rejected": -1.0087178945541382, |
| "logps/chosen": -297.63873291015625, |
| "logps/rejected": -330.9602966308594, |
| "loss": 0.5515, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.2629808187484741, |
| "rewards/margins": 0.5231103897094727, |
| "rewards/rejected": -0.7860912084579468, |
| "step": 360 |
| }, |
| { |
| "dpo_lambda": 0.9622641205787659, |
| "epoch": 0.7558230829625753, |
| "grad_norm": 19.06758201325056, |
| "learning_rate": 8.490561882286135e-08, |
| "logits/chosen": -1.0516997575759888, |
| "logits/rejected": -1.0711076259613037, |
| "logps/chosen": -349.70343017578125, |
| "logps/rejected": -354.0404052734375, |
| "loss": 0.536, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.26353931427001953, |
| "rewards/margins": 0.5523825883865356, |
| "rewards/rejected": -0.8159219026565552, |
| "step": 361 |
| }, |
| { |
| "dpo_lambda": 0.962159276008606, |
| "epoch": 0.7579167757131641, |
| "grad_norm": 26.94997496124403, |
| "learning_rate": 8.353527464267104e-08, |
| "logits/chosen": -0.9543567299842834, |
| "logits/rejected": -1.0147325992584229, |
| "logps/chosen": -305.9583740234375, |
| "logps/rejected": -365.9261779785156, |
| "loss": 0.5791, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.21509583294391632, |
| "rewards/margins": 0.5419927835464478, |
| "rewards/rejected": -0.7570887207984924, |
| "step": 362 |
| }, |
| { |
| "dpo_lambda": 0.9620543718338013, |
| "epoch": 0.760010468463753, |
| "grad_norm": 74.72502474786462, |
| "learning_rate": 8.217385746050742e-08, |
| "logits/chosen": -0.9676810503005981, |
| "logits/rejected": -1.0195143222808838, |
| "logps/chosen": -336.9450378417969, |
| "logps/rejected": -331.18798828125, |
| "loss": 0.5299, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.381142795085907, |
| "rewards/margins": 0.49130770564079285, |
| "rewards/rejected": -0.8724505305290222, |
| "step": 363 |
| }, |
| { |
| "dpo_lambda": 0.9619497656822205, |
| "epoch": 0.7621041612143418, |
| "grad_norm": 19.404892216299483, |
| "learning_rate": 8.082144028504231e-08, |
| "logits/chosen": -0.8616032600402832, |
| "logits/rejected": -0.9383154511451721, |
| "logps/chosen": -275.91143798828125, |
| "logps/rejected": -338.1697692871094, |
| "loss": 0.5035, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.23062482476234436, |
| "rewards/margins": 0.6644704341888428, |
| "rewards/rejected": -0.8950952887535095, |
| "step": 364 |
| }, |
| { |
| "dpo_lambda": 0.9618448615074158, |
| "epoch": 0.7641978539649307, |
| "grad_norm": 29.53717574810475, |
| "learning_rate": 7.947809564230445e-08, |
| "logits/chosen": -0.8835728764533997, |
| "logits/rejected": -0.9612337946891785, |
| "logps/chosen": -369.8661193847656, |
| "logps/rejected": -361.1076354980469, |
| "loss": 0.6037, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.3223067820072174, |
| "rewards/margins": 0.3532503843307495, |
| "rewards/rejected": -0.6755571961402893, |
| "step": 365 |
| }, |
| { |
| "dpo_lambda": 0.9617400169372559, |
| "epoch": 0.7662915467155195, |
| "grad_norm": 10.307277460270527, |
| "learning_rate": 7.814389557179016e-08, |
| "logits/chosen": -1.0233443975448608, |
| "logits/rejected": -1.0403273105621338, |
| "logps/chosen": -318.3058776855469, |
| "logps/rejected": -338.8187255859375, |
| "loss": 0.5634, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.3016853332519531, |
| "rewards/margins": 0.32519543170928955, |
| "rewards/rejected": -0.6268807649612427, |
| "step": 366 |
| }, |
| { |
| "dpo_lambda": 0.961635172367096, |
| "epoch": 0.7683852394661084, |
| "grad_norm": 45.00455227104742, |
| "learning_rate": 7.681891162260015e-08, |
| "logits/chosen": -0.9098859429359436, |
| "logits/rejected": -0.9500433206558228, |
| "logps/chosen": -321.8980712890625, |
| "logps/rejected": -321.658447265625, |
| "loss": 0.5075, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.3374080955982208, |
| "rewards/margins": 0.6366255283355713, |
| "rewards/rejected": -0.974033534526825, |
| "step": 367 |
| }, |
| { |
| "dpo_lambda": 0.961530327796936, |
| "epoch": 0.7704789322166972, |
| "grad_norm": 16.44474720560638, |
| "learning_rate": 7.550321484960251e-08, |
| "logits/chosen": -0.9235467314720154, |
| "logits/rejected": -0.9864405393600464, |
| "logps/chosen": -318.61566162109375, |
| "logps/rejected": -323.4438171386719, |
| "loss": 0.5266, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.1688789427280426, |
| "rewards/margins": 0.5078449845314026, |
| "rewards/rejected": -0.6767238974571228, |
| "step": 368 |
| }, |
| { |
| "dpo_lambda": 0.9614256620407104, |
| "epoch": 0.7725726249672861, |
| "grad_norm": 31.33862125443488, |
| "learning_rate": 7.419687580962222e-08, |
| "logits/chosen": -0.8263933658599854, |
| "logits/rejected": -0.8173232674598694, |
| "logps/chosen": -324.09283447265625, |
| "logps/rejected": -351.4126281738281, |
| "loss": 0.5795, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.42795392870903015, |
| "rewards/margins": 0.4751317799091339, |
| "rewards/rejected": -0.9030857086181641, |
| "step": 369 |
| }, |
| { |
| "dpo_lambda": 0.9613208174705505, |
| "epoch": 0.7746663177178749, |
| "grad_norm": 42.98613541209831, |
| "learning_rate": 7.289996455765748e-08, |
| "logits/chosen": -0.9016161561012268, |
| "logits/rejected": -0.9416869282722473, |
| "logps/chosen": -355.3125305175781, |
| "logps/rejected": -369.4179992675781, |
| "loss": 0.5451, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.31742802262306213, |
| "rewards/margins": 0.5414060354232788, |
| "rewards/rejected": -0.8588340878486633, |
| "step": 370 |
| }, |
| { |
| "dpo_lambda": 0.9612159132957458, |
| "epoch": 0.7767600104684638, |
| "grad_norm": 28.36509965644124, |
| "learning_rate": 7.161255064312283e-08, |
| "logits/chosen": -0.9476215243339539, |
| "logits/rejected": -0.9706940650939941, |
| "logps/chosen": -316.4869384765625, |
| "logps/rejected": -396.5313415527344, |
| "loss": 0.5328, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.3681057095527649, |
| "rewards/margins": 0.6468718647956848, |
| "rewards/rejected": -1.0149775743484497, |
| "step": 371 |
| }, |
| { |
| "dpo_lambda": 0.9611111283302307, |
| "epoch": 0.7788537032190526, |
| "grad_norm": 23.803503779777298, |
| "learning_rate": 7.033470310611945e-08, |
| "logits/chosen": -0.9440561532974243, |
| "logits/rejected": -1.0243254899978638, |
| "logps/chosen": -438.1816711425781, |
| "logps/rejected": -381.82623291015625, |
| "loss": 0.5516, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.37924379110336304, |
| "rewards/margins": 0.5478464365005493, |
| "rewards/rejected": -0.9270902276039124, |
| "step": 372 |
| }, |
| { |
| "dpo_lambda": 0.961006224155426, |
| "epoch": 0.7809473959696415, |
| "grad_norm": 27.72182838034728, |
| "learning_rate": 6.906649047373245e-08, |
| "logits/chosen": -0.9706982970237732, |
| "logits/rejected": -0.9654771089553833, |
| "logps/chosen": -333.77789306640625, |
| "logps/rejected": -391.4490966796875, |
| "loss": 0.5528, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.2363077700138092, |
| "rewards/margins": 0.6813668608665466, |
| "rewards/rejected": -0.9176746606826782, |
| "step": 373 |
| }, |
| { |
| "dpo_lambda": 0.9609013795852661, |
| "epoch": 0.7830410887202303, |
| "grad_norm": 20.681732016502796, |
| "learning_rate": 6.780798075635675e-08, |
| "logits/chosen": -0.9225718379020691, |
| "logits/rejected": -0.9658865928649902, |
| "logps/chosen": -353.7083435058594, |
| "logps/rejected": -369.9117431640625, |
| "loss": 0.5534, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.30636295676231384, |
| "rewards/margins": 0.6070533990859985, |
| "rewards/rejected": -0.9134163856506348, |
| "step": 374 |
| }, |
| { |
| "dpo_lambda": 0.9607967138290405, |
| "epoch": 0.7851347814708192, |
| "grad_norm": 31.186102107762363, |
| "learning_rate": 6.655924144404906e-08, |
| "logits/chosen": -0.8658726215362549, |
| "logits/rejected": -0.9505506157875061, |
| "logps/chosen": -310.1218566894531, |
| "logps/rejected": -356.1629943847656, |
| "loss": 0.5331, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.30431026220321655, |
| "rewards/margins": 0.6359937191009521, |
| "rewards/rejected": -0.9403039216995239, |
| "step": 375 |
| }, |
| { |
| "dpo_lambda": 0.9606918692588806, |
| "epoch": 0.787228474221408, |
| "grad_norm": 24.269420848253173, |
| "learning_rate": 6.532033950290885e-08, |
| "logits/chosen": -0.9355162382125854, |
| "logits/rejected": -0.9577205181121826, |
| "logps/chosen": -338.4344787597656, |
| "logps/rejected": -387.79913330078125, |
| "loss": 0.5694, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.39728814363479614, |
| "rewards/margins": 0.5328594446182251, |
| "rewards/rejected": -0.9301475882530212, |
| "step": 376 |
| }, |
| { |
| "dpo_lambda": 0.9605870246887207, |
| "epoch": 0.7893221669719969, |
| "grad_norm": 19.188064859196658, |
| "learning_rate": 6.409134137148736e-08, |
| "logits/chosen": -1.0201990604400635, |
| "logits/rejected": -1.047384262084961, |
| "logps/chosen": -343.03875732421875, |
| "logps/rejected": -315.65362548828125, |
| "loss": 0.5314, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.3237241506576538, |
| "rewards/margins": 0.49253687262535095, |
| "rewards/rejected": -0.8162609934806824, |
| "step": 377 |
| }, |
| { |
| "dpo_lambda": 0.9604821801185608, |
| "epoch": 0.7914158597225857, |
| "grad_norm": 24.57431465319421, |
| "learning_rate": 6.28723129572247e-08, |
| "logits/chosen": -0.8773887157440186, |
| "logits/rejected": -0.9270977973937988, |
| "logps/chosen": -334.6279296875, |
| "logps/rejected": -367.80572509765625, |
| "loss": 0.594, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.3744312524795532, |
| "rewards/margins": 0.42570799589157104, |
| "rewards/rejected": -0.800139307975769, |
| "step": 378 |
| }, |
| { |
| "dpo_lambda": 0.9603772759437561, |
| "epoch": 0.7935095524731746, |
| "grad_norm": 28.46042702704832, |
| "learning_rate": 6.166331963291519e-08, |
| "logits/chosen": -1.055816411972046, |
| "logits/rejected": -1.029515027999878, |
| "logps/chosen": -323.7863464355469, |
| "logps/rejected": -333.7519226074219, |
| "loss": 0.5726, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.4126008450984955, |
| "rewards/margins": 0.49342429637908936, |
| "rewards/rejected": -0.9060250520706177, |
| "step": 379 |
| }, |
| { |
| "dpo_lambda": 0.9602726697921753, |
| "epoch": 0.7956032452237635, |
| "grad_norm": 18.658115359359524, |
| "learning_rate": 6.046442623320145e-08, |
| "logits/chosen": -1.018285870552063, |
| "logits/rejected": -1.0573500394821167, |
| "logps/chosen": -330.10101318359375, |
| "logps/rejected": -346.8293151855469, |
| "loss": 0.5255, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.22925367951393127, |
| "rewards/margins": 0.6084834337234497, |
| "rewards/rejected": -0.8377372026443481, |
| "step": 380 |
| }, |
| { |
| "dpo_lambda": 0.9601677656173706, |
| "epoch": 0.7976969379743523, |
| "grad_norm": 16.408753713537546, |
| "learning_rate": 5.9275697051098275e-08, |
| "logits/chosen": -0.8530938625335693, |
| "logits/rejected": -0.9515883922576904, |
| "logps/chosen": -239.74586486816406, |
| "logps/rejected": -339.2502746582031, |
| "loss": 0.502, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.22224396467208862, |
| "rewards/margins": 0.6877234578132629, |
| "rewards/rejected": -0.9099674224853516, |
| "step": 381 |
| }, |
| { |
| "dpo_lambda": 0.9600629210472107, |
| "epoch": 0.7997906307249412, |
| "grad_norm": 21.903923323456592, |
| "learning_rate": 5.809719583454414e-08, |
| "logits/chosen": -0.9614814519882202, |
| "logits/rejected": -0.962985634803772, |
| "logps/chosen": -300.4097595214844, |
| "logps/rejected": -382.86370849609375, |
| "loss": 0.5261, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.26442432403564453, |
| "rewards/margins": 0.5285326242446899, |
| "rewards/rejected": -0.7929569482803345, |
| "step": 382 |
| }, |
| { |
| "dpo_lambda": 0.9599580764770508, |
| "epoch": 0.80188432347553, |
| "grad_norm": 27.69460396939364, |
| "learning_rate": 5.6928985782982524e-08, |
| "logits/chosen": -1.00211763381958, |
| "logits/rejected": -0.9333101511001587, |
| "logps/chosen": -371.8981628417969, |
| "logps/rejected": -389.7657165527344, |
| "loss": 0.5818, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.3752930164337158, |
| "rewards/margins": 0.4458085000514984, |
| "rewards/rejected": -0.8211015462875366, |
| "step": 383 |
| }, |
| { |
| "dpo_lambda": 0.9598532319068909, |
| "epoch": 0.8039780162261189, |
| "grad_norm": 15.880624776905542, |
| "learning_rate": 5.57711295439732e-08, |
| "logits/chosen": -1.0322285890579224, |
| "logits/rejected": -1.0425132513046265, |
| "logps/chosen": -356.7442626953125, |
| "logps/rejected": -409.0237121582031, |
| "loss": 0.5218, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.2565978765487671, |
| "rewards/margins": 0.6591013669967651, |
| "rewards/rejected": -0.9156992435455322, |
| "step": 384 |
| }, |
| { |
| "dpo_lambda": 0.959748387336731, |
| "epoch": 0.8060717089767077, |
| "grad_norm": 15.997055013777837, |
| "learning_rate": 5.4623689209832484e-08, |
| "logits/chosen": -0.9438518285751343, |
| "logits/rejected": -1.0183533430099487, |
| "logps/chosen": -339.08306884765625, |
| "logps/rejected": -395.9681396484375, |
| "loss": 0.5102, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.36264917254447937, |
| "rewards/margins": 0.6978291273117065, |
| "rewards/rejected": -1.0604783296585083, |
| "step": 385 |
| }, |
| { |
| "dpo_lambda": 0.9596434831619263, |
| "epoch": 0.8081654017272966, |
| "grad_norm": 21.110905175575006, |
| "learning_rate": 5.3486726314303175e-08, |
| "logits/chosen": -0.9347030520439148, |
| "logits/rejected": -0.9683570265769958, |
| "logps/chosen": -353.3894958496094, |
| "logps/rejected": -328.5455322265625, |
| "loss": 0.5475, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.2658626139163971, |
| "rewards/margins": 0.49515584111213684, |
| "rewards/rejected": -0.7610185146331787, |
| "step": 386 |
| }, |
| { |
| "dpo_lambda": 0.9595388174057007, |
| "epoch": 0.8102590944778854, |
| "grad_norm": 14.055777953199689, |
| "learning_rate": 5.2360301829254745e-08, |
| "logits/chosen": -0.9676686525344849, |
| "logits/rejected": -1.0681111812591553, |
| "logps/chosen": -349.0576171875, |
| "logps/rejected": -325.227294921875, |
| "loss": 0.5377, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.14794021844863892, |
| "rewards/margins": 0.6320769786834717, |
| "rewards/rejected": -0.7800171971321106, |
| "step": 387 |
| }, |
| { |
| "dpo_lambda": 0.9594339728355408, |
| "epoch": 0.8123527872284743, |
| "grad_norm": 37.7395918970165, |
| "learning_rate": 5.1244476161413806e-08, |
| "logits/chosen": -0.9613102078437805, |
| "logits/rejected": -0.9784658551216125, |
| "logps/chosen": -340.90447998046875, |
| "logps/rejected": -406.65533447265625, |
| "loss": 0.55, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.2638137936592102, |
| "rewards/margins": 0.6216176748275757, |
| "rewards/rejected": -0.8854314684867859, |
| "step": 388 |
| }, |
| { |
| "dpo_lambda": 0.9593291282653809, |
| "epoch": 0.814446479979063, |
| "grad_norm": 20.798435257784853, |
| "learning_rate": 5.013930914912476e-08, |
| "logits/chosen": -0.8479200601577759, |
| "logits/rejected": -0.9247003197669983, |
| "logps/chosen": -310.15374755859375, |
| "logps/rejected": -348.4466247558594, |
| "loss": 0.5478, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.3163961172103882, |
| "rewards/margins": 0.522079348564148, |
| "rewards/rejected": -0.8384754061698914, |
| "step": 389 |
| }, |
| { |
| "dpo_lambda": 0.959224283695221, |
| "epoch": 0.816540172729652, |
| "grad_norm": 54.64575176217907, |
| "learning_rate": 4.904486005914027e-08, |
| "logits/chosen": -0.9330431222915649, |
| "logits/rejected": -0.9549581408500671, |
| "logps/chosen": -365.50762939453125, |
| "logps/rejected": -346.893310546875, |
| "loss": 0.5593, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.45057371258735657, |
| "rewards/margins": 0.45801979303359985, |
| "rewards/rejected": -0.908593475818634, |
| "step": 390 |
| }, |
| { |
| "dpo_lambda": 0.959119439125061, |
| "epoch": 0.8186338654802408, |
| "grad_norm": 28.884583790867374, |
| "learning_rate": 4.796118758344353e-08, |
| "logits/chosen": -0.9886559247970581, |
| "logits/rejected": -1.0360742807388306, |
| "logps/chosen": -311.6133728027344, |
| "logps/rejected": -329.0717468261719, |
| "loss": 0.5356, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.3692110478878021, |
| "rewards/margins": 0.5127213001251221, |
| "rewards/rejected": -0.8819323182106018, |
| "step": 391 |
| }, |
| { |
| "dpo_lambda": 0.9590147733688354, |
| "epoch": 0.8207275582308297, |
| "grad_norm": 47.21987697553988, |
| "learning_rate": 4.688834983610082e-08, |
| "logits/chosen": -1.0307235717773438, |
| "logits/rejected": -0.9974226355552673, |
| "logps/chosen": -307.7640380859375, |
| "logps/rejected": -384.6656799316406, |
| "loss": 0.541, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.34472477436065674, |
| "rewards/margins": 0.5033468008041382, |
| "rewards/rejected": -0.8480715155601501, |
| "step": 392 |
| }, |
| { |
| "dpo_lambda": 0.9589099287986755, |
| "epoch": 0.8228212509814185, |
| "grad_norm": 36.144797883020956, |
| "learning_rate": 4.582640435014459e-08, |
| "logits/chosen": -1.0393571853637695, |
| "logits/rejected": -1.0646288394927979, |
| "logps/chosen": -303.0776672363281, |
| "logps/rejected": -308.91864013671875, |
| "loss": 0.5749, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.3313845694065094, |
| "rewards/margins": 0.454150915145874, |
| "rewards/rejected": -0.785535454750061, |
| "step": 393 |
| }, |
| { |
| "dpo_lambda": 0.9588050246238708, |
| "epoch": 0.8249149437320074, |
| "grad_norm": 28.405590539672186, |
| "learning_rate": 4.477540807448832e-08, |
| "logits/chosen": -0.8921625018119812, |
| "logits/rejected": -0.9136333465576172, |
| "logps/chosen": -355.0310363769531, |
| "logps/rejected": -426.3016662597656, |
| "loss": 0.5566, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.3053607940673828, |
| "rewards/margins": 0.6183335781097412, |
| "rewards/rejected": -0.9236944913864136, |
| "step": 394 |
| }, |
| { |
| "dpo_lambda": 0.9587001800537109, |
| "epoch": 0.8270086364825961, |
| "grad_norm": 15.539276399808447, |
| "learning_rate": 4.373541737087263e-08, |
| "logits/chosen": -1.0075271129608154, |
| "logits/rejected": -1.028421401977539, |
| "logps/chosen": -312.7923583984375, |
| "logps/rejected": -374.7745361328125, |
| "loss": 0.5639, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.3122669458389282, |
| "rewards/margins": 0.5935348868370056, |
| "rewards/rejected": -0.9058018922805786, |
| "step": 395 |
| }, |
| { |
| "dpo_lambda": 0.958595335483551, |
| "epoch": 0.829102329233185, |
| "grad_norm": 26.03734566553524, |
| "learning_rate": 4.270648801084295e-08, |
| "logits/chosen": -0.9819083213806152, |
| "logits/rejected": -0.9753702282905579, |
| "logps/chosen": -316.38751220703125, |
| "logps/rejected": -319.94915771484375, |
| "loss": 0.5703, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.39307281374931335, |
| "rewards/margins": 0.3845985233783722, |
| "rewards/rejected": -0.7776713967323303, |
| "step": 396 |
| }, |
| { |
| "dpo_lambda": 0.9584904909133911, |
| "epoch": 0.8311960219837738, |
| "grad_norm": 25.113826259403137, |
| "learning_rate": 4.168867517275806e-08, |
| "logits/chosen": -0.9145287871360779, |
| "logits/rejected": -0.9403222799301147, |
| "logps/chosen": -245.5500030517578, |
| "logps/rejected": -329.1048583984375, |
| "loss": 0.5393, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.36393922567367554, |
| "rewards/margins": 0.5811597108840942, |
| "rewards/rejected": -0.945098876953125, |
| "step": 397 |
| }, |
| { |
| "dpo_lambda": 0.9583855867385864, |
| "epoch": 0.8332897147343628, |
| "grad_norm": 44.51116164273478, |
| "learning_rate": 4.0682033438831584e-08, |
| "logits/chosen": -0.9946930408477783, |
| "logits/rejected": -0.9629707932472229, |
| "logps/chosen": -267.51715087890625, |
| "logps/rejected": -350.3848876953125, |
| "loss": 0.5696, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.40482285618782043, |
| "rewards/margins": 0.33537381887435913, |
| "rewards/rejected": -0.7401966452598572, |
| "step": 398 |
| }, |
| { |
| "dpo_lambda": 0.9582809805870056, |
| "epoch": 0.8353834074849515, |
| "grad_norm": 30.344521852168064, |
| "learning_rate": 3.968661679220467e-08, |
| "logits/chosen": -0.9283524751663208, |
| "logits/rejected": -0.9831556081771851, |
| "logps/chosen": -384.8466491699219, |
| "logps/rejected": -402.1518249511719, |
| "loss": 0.5402, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.4025726616382599, |
| "rewards/margins": 0.5733284950256348, |
| "rewards/rejected": -0.9759011268615723, |
| "step": 399 |
| }, |
| { |
| "dpo_lambda": 0.9581760764122009, |
| "epoch": 0.8374771002355405, |
| "grad_norm": 44.88339103217458, |
| "learning_rate": 3.8702478614051345e-08, |
| "logits/chosen": -0.9853564500808716, |
| "logits/rejected": -0.9621061086654663, |
| "logps/chosen": -376.42376708984375, |
| "logps/rejected": -432.05963134765625, |
| "loss": 0.5423, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.31213808059692383, |
| "rewards/margins": 0.6543071866035461, |
| "rewards/rejected": -0.96644526720047, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.8374771002355405, |
| "eval_dpo_lambda": 0.9580713510513306, |
| "eval_logits/chosen": -0.9665474891662598, |
| "eval_logits/rejected": -1.0032734870910645, |
| "eval_logps/chosen": -339.02130126953125, |
| "eval_logps/rejected": -360.9908142089844, |
| "eval_loss": 0.561345636844635, |
| "eval_rewards/accuracies": 0.7289999723434448, |
| "eval_rewards/chosen": -0.38374418020248413, |
| "eval_rewards/margins": 0.5158559679985046, |
| "eval_rewards/rejected": -0.899600088596344, |
| "eval_runtime": 560.9381, |
| "eval_samples_per_second": 3.565, |
| "eval_steps_per_second": 0.891, |
| "step": 400 |
| }, |
| { |
| "dpo_lambda": 0.9580712914466858, |
| "epoch": 0.8395707929861292, |
| "grad_norm": 82.15355464781004, |
| "learning_rate": 3.772967168071517e-08, |
| "logits/chosen": -1.0035700798034668, |
| "logits/rejected": -1.0144524574279785, |
| "logps/chosen": -322.69830322265625, |
| "logps/rejected": -336.5651550292969, |
| "loss": 0.5728, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.3935065269470215, |
| "rewards/margins": 0.46786949038505554, |
| "rewards/rejected": -0.8613760471343994, |
| "step": 401 |
| }, |
| { |
| "dpo_lambda": 0.9579663872718811, |
| "epoch": 0.8416644857367181, |
| "grad_norm": 20.35226428221194, |
| "learning_rate": 3.676824816087978e-08, |
| "logits/chosen": -0.9380530714988708, |
| "logits/rejected": -0.9650043845176697, |
| "logps/chosen": -355.93719482421875, |
| "logps/rejected": -356.6396789550781, |
| "loss": 0.5223, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.19415956735610962, |
| "rewards/margins": 0.6138423681259155, |
| "rewards/rejected": -0.8080019950866699, |
| "step": 402 |
| }, |
| { |
| "dpo_lambda": 0.9578615427017212, |
| "epoch": 0.8437581784873069, |
| "grad_norm": 42.29350910621693, |
| "learning_rate": 3.581825961277074e-08, |
| "logits/chosen": -0.9577827453613281, |
| "logits/rejected": -1.0395058393478394, |
| "logps/chosen": -357.50103759765625, |
| "logps/rejected": -378.7171630859375, |
| "loss": 0.5769, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.3822963237762451, |
| "rewards/margins": 0.4797073006629944, |
| "rewards/rejected": -0.8620035648345947, |
| "step": 403 |
| }, |
| { |
| "dpo_lambda": 0.9577568769454956, |
| "epoch": 0.8458518712378958, |
| "grad_norm": 21.139864207028474, |
| "learning_rate": 3.487975698139084e-08, |
| "logits/chosen": -1.053609848022461, |
| "logits/rejected": -1.046288013458252, |
| "logps/chosen": -348.7900085449219, |
| "logps/rejected": -376.8426208496094, |
| "loss": 0.5609, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.4225596487522125, |
| "rewards/margins": 0.3801911175251007, |
| "rewards/rejected": -0.802750825881958, |
| "step": 404 |
| }, |
| { |
| "dpo_lambda": 0.9576520323753357, |
| "epoch": 0.8479455639884846, |
| "grad_norm": 48.23981949574903, |
| "learning_rate": 3.3952790595787986e-08, |
| "logits/chosen": -1.039850115776062, |
| "logits/rejected": -1.068182349205017, |
| "logps/chosen": -318.3228759765625, |
| "logps/rejected": -346.5895080566406, |
| "loss": 0.522, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.4849662184715271, |
| "rewards/margins": 0.4318442940711975, |
| "rewards/rejected": -0.9168104529380798, |
| "step": 405 |
| }, |
| { |
| "dpo_lambda": 0.9575471878051758, |
| "epoch": 0.8500392567390735, |
| "grad_norm": 27.516331980321333, |
| "learning_rate": 3.303741016635614e-08, |
| "logits/chosen": -1.0031503438949585, |
| "logits/rejected": -1.0047969818115234, |
| "logps/chosen": -370.5401306152344, |
| "logps/rejected": -397.98040771484375, |
| "loss": 0.557, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.27663034200668335, |
| "rewards/margins": 0.6528828144073486, |
| "rewards/rejected": -0.9295130372047424, |
| "step": 406 |
| }, |
| { |
| "dpo_lambda": 0.9574423432350159, |
| "epoch": 0.8521329494896623, |
| "grad_norm": 20.04658083953901, |
| "learning_rate": 3.2133664782169944e-08, |
| "logits/chosen": -0.975938081741333, |
| "logits/rejected": -0.956437349319458, |
| "logps/chosen": -325.4172058105469, |
| "logps/rejected": -350.14862060546875, |
| "loss": 0.523, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.40274572372436523, |
| "rewards/margins": 0.6021479964256287, |
| "rewards/rejected": -1.0048936605453491, |
| "step": 407 |
| }, |
| { |
| "dpo_lambda": 0.9573374390602112, |
| "epoch": 0.8542266422402512, |
| "grad_norm": 28.927907054098547, |
| "learning_rate": 3.12416029083514e-08, |
| "logits/chosen": -0.9998895525932312, |
| "logits/rejected": -1.0152561664581299, |
| "logps/chosen": -333.8466796875, |
| "logps/rejected": -372.66162109375, |
| "loss": 0.5512, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.42361342906951904, |
| "rewards/margins": 0.4960295557975769, |
| "rewards/rejected": -0.919642984867096, |
| "step": 408 |
| }, |
| { |
| "dpo_lambda": 0.9572325944900513, |
| "epoch": 0.8563203349908401, |
| "grad_norm": 28.95623360194244, |
| "learning_rate": 3.036127238347164e-08, |
| "logits/chosen": -1.0345758199691772, |
| "logits/rejected": -1.0678658485412598, |
| "logps/chosen": -393.51934814453125, |
| "logps/rejected": -419.20819091796875, |
| "loss": 0.5428, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.3088226020336151, |
| "rewards/margins": 0.5775026082992554, |
| "rewards/rejected": -0.8863251805305481, |
| "step": 409 |
| }, |
| { |
| "dpo_lambda": 0.9571279287338257, |
| "epoch": 0.8584140277414289, |
| "grad_norm": 38.71222399422127, |
| "learning_rate": 2.9492720416985e-08, |
| "logits/chosen": -0.9824857115745544, |
| "logits/rejected": -0.9625248908996582, |
| "logps/chosen": -327.7845764160156, |
| "logps/rejected": -384.799560546875, |
| "loss": 0.5879, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.3323441743850708, |
| "rewards/margins": 0.6053525805473328, |
| "rewards/rejected": -0.937696635723114, |
| "step": 410 |
| }, |
| { |
| "dpo_lambda": 0.9570230841636658, |
| "epoch": 0.8605077204920178, |
| "grad_norm": 63.673739836807414, |
| "learning_rate": 2.863599358669755e-08, |
| "logits/chosen": -1.0002498626708984, |
| "logits/rejected": -0.9433389902114868, |
| "logps/chosen": -332.62017822265625, |
| "logps/rejected": -383.5394592285156, |
| "loss": 0.5866, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.4152870178222656, |
| "rewards/margins": 0.41510239243507385, |
| "rewards/rejected": -0.8303893804550171, |
| "step": 411 |
| }, |
| { |
| "dpo_lambda": 0.9569182395935059, |
| "epoch": 0.8626014132426066, |
| "grad_norm": 47.031673049240815, |
| "learning_rate": 2.7791137836269158e-08, |
| "logits/chosen": -0.9564170241355896, |
| "logits/rejected": -0.9423116445541382, |
| "logps/chosen": -329.16876220703125, |
| "logps/rejected": -396.54949951171875, |
| "loss": 0.5372, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.37793248891830444, |
| "rewards/margins": 0.6080735325813293, |
| "rewards/rejected": -0.986005961894989, |
| "step": 412 |
| }, |
| { |
| "dpo_lambda": 0.956813395023346, |
| "epoch": 0.8646951059931955, |
| "grad_norm": 20.115652999297403, |
| "learning_rate": 2.6958198472749717e-08, |
| "logits/chosen": -0.993791937828064, |
| "logits/rejected": -1.0071156024932861, |
| "logps/chosen": -288.9036560058594, |
| "logps/rejected": -332.8777770996094, |
| "loss": 0.5507, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.34835270047187805, |
| "rewards/margins": 0.5191000699996948, |
| "rewards/rejected": -0.8674527406692505, |
| "step": 413 |
| }, |
| { |
| "dpo_lambda": 0.956708550453186, |
| "epoch": 0.8667887987437843, |
| "grad_norm": 28.643664444242688, |
| "learning_rate": 2.613722016414943e-08, |
| "logits/chosen": -0.9087300896644592, |
| "logits/rejected": -0.9807073473930359, |
| "logps/chosen": -331.19329833984375, |
| "logps/rejected": -377.75054931640625, |
| "loss": 0.5429, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.48658519983291626, |
| "rewards/margins": 0.5433869361877441, |
| "rewards/rejected": -1.0299721956253052, |
| "step": 414 |
| }, |
| { |
| "dpo_lambda": 0.9566038846969604, |
| "epoch": 0.8688824914943732, |
| "grad_norm": 25.510442310385585, |
| "learning_rate": 2.5328246937043525e-08, |
| "logits/chosen": -0.9275650382041931, |
| "logits/rejected": -0.966849684715271, |
| "logps/chosen": -344.0802001953125, |
| "logps/rejected": -348.220947265625, |
| "loss": 0.5552, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.34558096528053284, |
| "rewards/margins": 0.6371087431907654, |
| "rewards/rejected": -0.982689619064331, |
| "step": 415 |
| }, |
| { |
| "dpo_lambda": 0.9564989805221558, |
| "epoch": 0.870976184244962, |
| "grad_norm": 19.797185961582827, |
| "learning_rate": 2.4531322174210973e-08, |
| "logits/chosen": -0.9148194789886475, |
| "logits/rejected": -1.0240569114685059, |
| "logps/chosen": -363.20098876953125, |
| "logps/rejected": -388.9029235839844, |
| "loss": 0.5692, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.2716209888458252, |
| "rewards/margins": 0.8080213665962219, |
| "rewards/rejected": -1.079642415046692, |
| "step": 416 |
| }, |
| { |
| "dpo_lambda": 0.9563941359519958, |
| "epoch": 0.8730698769955509, |
| "grad_norm": 37.76417428426363, |
| "learning_rate": 2.3746488612308295e-08, |
| "logits/chosen": -0.9347919225692749, |
| "logits/rejected": -0.9365442395210266, |
| "logps/chosen": -319.30450439453125, |
| "logps/rejected": -377.2983703613281, |
| "loss": 0.5504, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.5698833465576172, |
| "rewards/margins": 0.35962584614753723, |
| "rewards/rejected": -0.9295092225074768, |
| "step": 417 |
| }, |
| { |
| "dpo_lambda": 0.9562892913818359, |
| "epoch": 0.8751635697461397, |
| "grad_norm": 30.23045274143435, |
| "learning_rate": 2.297378833957761e-08, |
| "logits/chosen": -1.0036827325820923, |
| "logits/rejected": -0.954819917678833, |
| "logps/chosen": -329.52276611328125, |
| "logps/rejected": -358.5249938964844, |
| "loss": 0.5439, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.37042689323425293, |
| "rewards/margins": 0.5442904829978943, |
| "rewards/rejected": -0.914717435836792, |
| "step": 418 |
| }, |
| { |
| "dpo_lambda": 0.956184446811676, |
| "epoch": 0.8772572624967286, |
| "grad_norm": 17.865199078300027, |
| "learning_rate": 2.2213262793589482e-08, |
| "logits/chosen": -0.9423766136169434, |
| "logits/rejected": -1.0231984853744507, |
| "logps/chosen": -332.6385192871094, |
| "logps/rejected": -357.6366882324219, |
| "loss": 0.5404, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.3678462505340576, |
| "rewards/margins": 0.5428791046142578, |
| "rewards/rejected": -0.9107253551483154, |
| "step": 419 |
| }, |
| { |
| "dpo_lambda": 0.9560796022415161, |
| "epoch": 0.8793509552473174, |
| "grad_norm": 12.8919686565624, |
| "learning_rate": 2.1464952759020856e-08, |
| "logits/chosen": -0.8776789903640747, |
| "logits/rejected": -0.9112561941146851, |
| "logps/chosen": -362.9369812011719, |
| "logps/rejected": -445.8304748535156, |
| "loss": 0.5028, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.24908962845802307, |
| "rewards/margins": 0.7293524146080017, |
| "rewards/rejected": -0.9784420728683472, |
| "step": 420 |
| }, |
| { |
| "dpo_lambda": 0.9559746980667114, |
| "epoch": 0.8814446479979063, |
| "grad_norm": 17.459827778455995, |
| "learning_rate": 2.07288983654679e-08, |
| "logits/chosen": -0.9969652891159058, |
| "logits/rejected": -1.0456459522247314, |
| "logps/chosen": -284.34271240234375, |
| "logps/rejected": -328.07318115234375, |
| "loss": 0.4952, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.37537533044815063, |
| "rewards/margins": 0.6198078989982605, |
| "rewards/rejected": -0.9951832294464111, |
| "step": 421 |
| }, |
| { |
| "dpo_lambda": 0.9558700919151306, |
| "epoch": 0.8835383407484951, |
| "grad_norm": 43.84604357896676, |
| "learning_rate": 2.0005139085293942e-08, |
| "logits/chosen": -0.9257749319076538, |
| "logits/rejected": -0.8814691305160522, |
| "logps/chosen": -320.6876220703125, |
| "logps/rejected": -344.1969299316406, |
| "loss": 0.5525, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.41354262828826904, |
| "rewards/margins": 0.4604254961013794, |
| "rewards/rejected": -0.8739681839942932, |
| "step": 422 |
| }, |
| { |
| "dpo_lambda": 0.9557651877403259, |
| "epoch": 0.885632033499084, |
| "grad_norm": 46.36867747989169, |
| "learning_rate": 1.9293713731512673e-08, |
| "logits/chosen": -0.9757592082023621, |
| "logits/rejected": -0.9672430157661438, |
| "logps/chosen": -335.0474853515625, |
| "logps/rejected": -392.3810119628906, |
| "loss": 0.5251, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.2871450185775757, |
| "rewards/margins": 0.6362358331680298, |
| "rewards/rejected": -0.9233807921409607, |
| "step": 423 |
| }, |
| { |
| "dpo_lambda": 0.955660343170166, |
| "epoch": 0.8877257262496728, |
| "grad_norm": 34.23218298531432, |
| "learning_rate": 1.8594660455706763e-08, |
| "logits/chosen": -0.8614395260810852, |
| "logits/rejected": -0.9894289970397949, |
| "logps/chosen": -327.67840576171875, |
| "logps/rejected": -366.784423828125, |
| "loss": 0.5035, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.3272741734981537, |
| "rewards/margins": 0.7001060247421265, |
| "rewards/rejected": -1.0273802280426025, |
| "step": 424 |
| }, |
| { |
| "dpo_lambda": 0.9555554986000061, |
| "epoch": 0.8898194190002617, |
| "grad_norm": 16.45766662860836, |
| "learning_rate": 1.7908016745981856e-08, |
| "logits/chosen": -0.9176836013793945, |
| "logits/rejected": -0.9757386445999146, |
| "logps/chosen": -409.4639587402344, |
| "logps/rejected": -359.6163330078125, |
| "loss": 0.5317, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.19637420773506165, |
| "rewards/margins": 0.7027495503425598, |
| "rewards/rejected": -0.8991237878799438, |
| "step": 425 |
| }, |
| { |
| "dpo_lambda": 0.9554506540298462, |
| "epoch": 0.8919131117508505, |
| "grad_norm": 38.79029297459658, |
| "learning_rate": 1.7233819424956247e-08, |
| "logits/chosen": -0.9371305108070374, |
| "logits/rejected": -0.9625764489173889, |
| "logps/chosen": -356.5591735839844, |
| "logps/rejected": -382.1607666015625, |
| "loss": 0.5267, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.34867197275161743, |
| "rewards/margins": 0.5435488224029541, |
| "rewards/rejected": -0.8922207951545715, |
| "step": 426 |
| }, |
| { |
| "dpo_lambda": 0.9553459882736206, |
| "epoch": 0.8940068045014394, |
| "grad_norm": 39.594727805679064, |
| "learning_rate": 1.6572104647786245e-08, |
| "logits/chosen": -1.0280911922454834, |
| "logits/rejected": -1.0318591594696045, |
| "logps/chosen": -322.733154296875, |
| "logps/rejected": -334.4246826171875, |
| "loss": 0.549, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.39758163690567017, |
| "rewards/margins": 0.5534409284591675, |
| "rewards/rejected": -0.9510226249694824, |
| "step": 427 |
| }, |
| { |
| "dpo_lambda": 0.9552411437034607, |
| "epoch": 0.8961004972520282, |
| "grad_norm": 12.049727091764039, |
| "learning_rate": 1.5922907900227017e-08, |
| "logits/chosen": -0.9746778011322021, |
| "logits/rejected": -1.0556857585906982, |
| "logps/chosen": -374.712646484375, |
| "logps/rejected": -401.69976806640625, |
| "loss": 0.5366, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.30862295627593994, |
| "rewards/margins": 0.7240450978279114, |
| "rewards/rejected": -1.0326679944992065, |
| "step": 428 |
| }, |
| { |
| "dpo_lambda": 0.955136239528656, |
| "epoch": 0.8981941900026171, |
| "grad_norm": 13.2303060377412, |
| "learning_rate": 1.5286263996730026e-08, |
| "logits/chosen": -0.9656727313995361, |
| "logits/rejected": -0.9746390581130981, |
| "logps/chosen": -362.42327880859375, |
| "logps/rejected": -377.53375244140625, |
| "loss": 0.5027, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -0.19976939260959625, |
| "rewards/margins": 0.6579819917678833, |
| "rewards/rejected": -0.8577514290809631, |
| "step": 429 |
| }, |
| { |
| "dpo_lambda": 0.9550314545631409, |
| "epoch": 0.9002878827532059, |
| "grad_norm": 18.80666324691916, |
| "learning_rate": 1.4662207078575684e-08, |
| "logits/chosen": -0.9734973907470703, |
| "logits/rejected": -1.0407707691192627, |
| "logps/chosen": -289.5849609375, |
| "logps/rejected": -310.9867858886719, |
| "loss": 0.5709, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.3958791494369507, |
| "rewards/margins": 0.4667404592037201, |
| "rewards/rejected": -0.8626196980476379, |
| "step": 430 |
| }, |
| { |
| "dpo_lambda": 0.9549265503883362, |
| "epoch": 0.9023815755037948, |
| "grad_norm": 21.860909121869923, |
| "learning_rate": 1.40507706120426e-08, |
| "logits/chosen": -1.0206691026687622, |
| "logits/rejected": -0.9979550242424011, |
| "logps/chosen": -302.2188720703125, |
| "logps/rejected": -331.165283203125, |
| "loss": 0.5281, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.3756422698497772, |
| "rewards/margins": 0.45793285965919495, |
| "rewards/rejected": -0.8335750699043274, |
| "step": 431 |
| }, |
| { |
| "dpo_lambda": 0.9548217058181763, |
| "epoch": 0.9044752682543836, |
| "grad_norm": 20.022550700009774, |
| "learning_rate": 1.345198738661285e-08, |
| "logits/chosen": -0.9343916177749634, |
| "logits/rejected": -0.9401783347129822, |
| "logps/chosen": -302.5766906738281, |
| "logps/rejected": -372.9184265136719, |
| "loss": 0.5767, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.39758020639419556, |
| "rewards/margins": 0.5371288061141968, |
| "rewards/rejected": -0.9347091317176819, |
| "step": 432 |
| }, |
| { |
| "dpo_lambda": 0.9547170400619507, |
| "epoch": 0.9065689610049725, |
| "grad_norm": 22.247403237563105, |
| "learning_rate": 1.2865889513213628e-08, |
| "logits/chosen": -0.8874664306640625, |
| "logits/rejected": -0.8951385617256165, |
| "logps/chosen": -334.44207763671875, |
| "logps/rejected": -412.373291015625, |
| "loss": 0.5207, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -0.30106163024902344, |
| "rewards/margins": 0.7014852166175842, |
| "rewards/rejected": -1.0025469064712524, |
| "step": 433 |
| }, |
| { |
| "dpo_lambda": 0.9546121954917908, |
| "epoch": 0.9086626537555613, |
| "grad_norm": 20.122232158028673, |
| "learning_rate": 1.2292508422495157e-08, |
| "logits/chosen": -0.905904233455658, |
| "logits/rejected": -0.977434515953064, |
| "logps/chosen": -324.328125, |
| "logps/rejected": -333.3005676269531, |
| "loss": 0.551, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.26499003171920776, |
| "rewards/margins": 0.5595492124557495, |
| "rewards/rejected": -0.8245391845703125, |
| "step": 434 |
| }, |
| { |
| "dpo_lambda": 0.9545073509216309, |
| "epoch": 0.9107563465061502, |
| "grad_norm": 173.3897449087187, |
| "learning_rate": 1.1731874863145142e-08, |
| "logits/chosen": -1.058231234550476, |
| "logits/rejected": -1.04081392288208, |
| "logps/chosen": -298.2507629394531, |
| "logps/rejected": -354.2373046875, |
| "loss": 0.572, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.3913850784301758, |
| "rewards/margins": 0.42320144176483154, |
| "rewards/rejected": -0.8145864605903625, |
| "step": 435 |
| }, |
| { |
| "dpo_lambda": 0.954402506351471, |
| "epoch": 0.912850039256739, |
| "grad_norm": 147.39545680366115, |
| "learning_rate": 1.118401890024001e-08, |
| "logits/chosen": -1.000075101852417, |
| "logits/rejected": -1.0360493659973145, |
| "logps/chosen": -382.035888671875, |
| "logps/rejected": -419.6065673828125, |
| "loss": 0.5386, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.40097954869270325, |
| "rewards/margins": 0.5406019687652588, |
| "rewards/rejected": -0.9415814876556396, |
| "step": 436 |
| }, |
| { |
| "dpo_lambda": 0.9542976021766663, |
| "epoch": 0.9149437320073279, |
| "grad_norm": 21.217270364197876, |
| "learning_rate": 1.06489699136324e-08, |
| "logits/chosen": -0.9560334086418152, |
| "logits/rejected": -1.035660743713379, |
| "logps/chosen": -325.727783203125, |
| "logps/rejected": -312.07403564453125, |
| "loss": 0.5881, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.39235901832580566, |
| "rewards/margins": 0.48345211148262024, |
| "rewards/rejected": -0.8758111000061035, |
| "step": 437 |
| }, |
| { |
| "dpo_lambda": 0.9541929960250854, |
| "epoch": 0.9170374247579168, |
| "grad_norm": 20.559866554121044, |
| "learning_rate": 1.0126756596375685e-08, |
| "logits/chosen": -0.9488348960876465, |
| "logits/rejected": -0.9916976690292358, |
| "logps/chosen": -308.44549560546875, |
| "logps/rejected": -326.90802001953125, |
| "loss": 0.5357, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.28534749150276184, |
| "rewards/margins": 0.6132487654685974, |
| "rewards/rejected": -0.8985961675643921, |
| "step": 438 |
| }, |
| { |
| "dpo_lambda": 0.9540880918502808, |
| "epoch": 0.9191311175085056, |
| "grad_norm": 20.55816599168075, |
| "learning_rate": 9.617406953185136e-09, |
| "logits/chosen": -0.9830228686332703, |
| "logits/rejected": -1.0618979930877686, |
| "logps/chosen": -367.99737548828125, |
| "logps/rejected": -469.106689453125, |
| "loss": 0.5231, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -0.24455446004867554, |
| "rewards/margins": 0.8762832283973694, |
| "rewards/rejected": -1.120837688446045, |
| "step": 439 |
| }, |
| { |
| "dpo_lambda": 0.9539832472801208, |
| "epoch": 0.9212248102590945, |
| "grad_norm": 28.839706619333853, |
| "learning_rate": 9.12094829893642e-09, |
| "logits/chosen": -1.036233901977539, |
| "logits/rejected": -0.9664689302444458, |
| "logps/chosen": -367.5104064941406, |
| "logps/rejected": -434.5174560546875, |
| "loss": 0.5366, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.2284281998872757, |
| "rewards/margins": 0.7398945093154907, |
| "rewards/rejected": -0.96832275390625, |
| "step": 440 |
| }, |
| { |
| "dpo_lambda": 0.9538784027099609, |
| "epoch": 0.9233185030096833, |
| "grad_norm": 31.28605342903596, |
| "learning_rate": 8.637407257200496e-09, |
| "logits/chosen": -0.89495849609375, |
| "logits/rejected": -0.8919556736946106, |
| "logps/chosen": -318.3620300292969, |
| "logps/rejected": -347.7049255371094, |
| "loss": 0.5517, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.36943739652633667, |
| "rewards/margins": 0.465165376663208, |
| "rewards/rejected": -0.8346028327941895, |
| "step": 441 |
| }, |
| { |
| "dpo_lambda": 0.953773558139801, |
| "epoch": 0.9254121957602722, |
| "grad_norm": 33.18324707086255, |
| "learning_rate": 8.166809758815895e-09, |
| "logits/chosen": -1.043076992034912, |
| "logits/rejected": -1.0556950569152832, |
| "logps/chosen": -394.7267150878906, |
| "logps/rejected": -431.6220703125, |
| "loss": 0.5411, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.4038960337638855, |
| "rewards/margins": 0.5986083149909973, |
| "rewards/rejected": -1.0025043487548828, |
| "step": 442 |
| }, |
| { |
| "dpo_lambda": 0.9536687135696411, |
| "epoch": 0.927505888510861, |
| "grad_norm": 51.68212005214359, |
| "learning_rate": 7.709181040498253e-09, |
| "logits/chosen": -0.960411787033081, |
| "logits/rejected": -0.9813458323478699, |
| "logps/chosen": -330.04736328125, |
| "logps/rejected": -385.2778015136719, |
| "loss": 0.5518, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.2757166922092438, |
| "rewards/margins": 0.5993854999542236, |
| "rewards/rejected": -0.8751022219657898, |
| "step": 443 |
| }, |
| { |
| "dpo_lambda": 0.9535638093948364, |
| "epoch": 0.9295995812614499, |
| "grad_norm": 19.114821707183086, |
| "learning_rate": 7.2645456434869965e-09, |
| "logits/chosen": -0.9486753344535828, |
| "logits/rejected": -1.034155011177063, |
| "logps/chosen": -294.3712158203125, |
| "logps/rejected": -359.9336242675781, |
| "loss": 0.57, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.41555386781692505, |
| "rewards/margins": 0.40340498089790344, |
| "rewards/rejected": -0.8189587593078613, |
| "step": 444 |
| }, |
| { |
| "dpo_lambda": 0.9534591436386108, |
| "epoch": 0.9316932740120387, |
| "grad_norm": 29.115280986236147, |
| "learning_rate": 6.832927412229017e-09, |
| "logits/chosen": -1.0227396488189697, |
| "logits/rejected": -1.0331047773361206, |
| "logps/chosen": -310.11004638671875, |
| "logps/rejected": -339.3343811035156, |
| "loss": 0.5575, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.2876787781715393, |
| "rewards/margins": 0.5395625829696655, |
| "rewards/rejected": -0.8272414207458496, |
| "step": 445 |
| }, |
| { |
| "dpo_lambda": 0.9533542990684509, |
| "epoch": 0.9337869667626276, |
| "grad_norm": 15.00058131136615, |
| "learning_rate": 6.414349493100129e-09, |
| "logits/chosen": -0.9562291502952576, |
| "logits/rejected": -0.9506933093070984, |
| "logps/chosen": -337.495849609375, |
| "logps/rejected": -396.34637451171875, |
| "loss": 0.5277, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.23384301364421844, |
| "rewards/margins": 0.6402769684791565, |
| "rewards/rejected": -0.8741199970245361, |
| "step": 446 |
| }, |
| { |
| "dpo_lambda": 0.953249454498291, |
| "epoch": 0.9358806595132164, |
| "grad_norm": 16.07486927161977, |
| "learning_rate": 6.0088343331638756e-09, |
| "logits/chosen": -0.9611780047416687, |
| "logits/rejected": -0.934112548828125, |
| "logps/chosen": -322.6107482910156, |
| "logps/rejected": -392.5720520019531, |
| "loss": 0.5271, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.3303907811641693, |
| "rewards/margins": 0.6360582709312439, |
| "rewards/rejected": -0.9664490818977356, |
| "step": 447 |
| }, |
| { |
| "dpo_lambda": 0.9531446099281311, |
| "epoch": 0.9379743522638053, |
| "grad_norm": 29.673921637972377, |
| "learning_rate": 5.616403678967624e-09, |
| "logits/chosen": -1.0144095420837402, |
| "logits/rejected": -1.026979684829712, |
| "logps/chosen": -349.0781555175781, |
| "logps/rejected": -386.89556884765625, |
| "loss": 0.539, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.4534134566783905, |
| "rewards/margins": 0.6086537837982178, |
| "rewards/rejected": -1.0620671510696411, |
| "step": 448 |
| }, |
| { |
| "dpo_lambda": 0.9530397653579712, |
| "epoch": 0.9400680450143941, |
| "grad_norm": 14.254234329687867, |
| "learning_rate": 5.2370785753763356e-09, |
| "logits/chosen": -0.8938382267951965, |
| "logits/rejected": -0.9977080821990967, |
| "logps/chosen": -288.8353271484375, |
| "logps/rejected": -345.09039306640625, |
| "loss": 0.5622, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.34774893522262573, |
| "rewards/margins": 0.44677218794822693, |
| "rewards/rejected": -0.7945210933685303, |
| "step": 449 |
| }, |
| { |
| "dpo_lambda": 0.9529350996017456, |
| "epoch": 0.942161737764983, |
| "grad_norm": 44.196200512016574, |
| "learning_rate": 4.8708793644441086e-09, |
| "logits/chosen": -0.8911122679710388, |
| "logits/rejected": -0.9161196351051331, |
| "logps/chosen": -303.7921142578125, |
| "logps/rejected": -349.2629699707031, |
| "loss": 0.5357, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.26537635922431946, |
| "rewards/margins": 0.6387584209442139, |
| "rewards/rejected": -0.9041347503662109, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.942161737764983, |
| "eval_dpo_lambda": 0.9528301954269409, |
| "eval_logits/chosen": -0.9672125577926636, |
| "eval_logits/rejected": -1.003003478050232, |
| "eval_logps/chosen": -338.48345947265625, |
| "eval_logps/rejected": -360.6006164550781, |
| "eval_loss": 0.561907947063446, |
| "eval_rewards/accuracies": 0.7310000061988831, |
| "eval_rewards/chosen": -0.3783654570579529, |
| "eval_rewards/margins": 0.517332911491394, |
| "eval_rewards/rejected": -0.8956983685493469, |
| "eval_runtime": 561.384, |
| "eval_samples_per_second": 3.563, |
| "eval_steps_per_second": 0.891, |
| "step": 450 |
| }, |
| { |
| "dpo_lambda": 0.9528302550315857, |
| "epoch": 0.9442554305155718, |
| "grad_norm": 15.699215737825678, |
| "learning_rate": 4.517825684323323e-09, |
| "logits/chosen": -0.9667340517044067, |
| "logits/rejected": -0.9940100312232971, |
| "logps/chosen": -347.5166015625, |
| "logps/rejected": -398.2073669433594, |
| "loss": 0.5401, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.36576253175735474, |
| "rewards/margins": 0.6338947415351868, |
| "rewards/rejected": -0.9996572732925415, |
| "step": 451 |
| }, |
| { |
| "dpo_lambda": 0.952725350856781, |
| "epoch": 0.9463491232661607, |
| "grad_norm": 73.7455655355022, |
| "learning_rate": 4.1779364682113794e-09, |
| "logits/chosen": -0.916843593120575, |
| "logits/rejected": -0.9902130365371704, |
| "logps/chosen": -378.66424560546875, |
| "logps/rejected": -383.3100280761719, |
| "loss": 0.6002, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.4185040593147278, |
| "rewards/margins": 0.43680867552757263, |
| "rewards/rejected": -0.8553128242492676, |
| "step": 452 |
| }, |
| { |
| "dpo_lambda": 0.9526205062866211, |
| "epoch": 0.9484428160167495, |
| "grad_norm": 19.729553777174523, |
| "learning_rate": 3.851229943335393e-09, |
| "logits/chosen": -0.9096695184707642, |
| "logits/rejected": -0.9353246688842773, |
| "logps/chosen": -337.8310852050781, |
| "logps/rejected": -331.5688781738281, |
| "loss": 0.5197, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.29002678394317627, |
| "rewards/margins": 0.6455761194229126, |
| "rewards/rejected": -0.9356027841567993, |
| "step": 453 |
| }, |
| { |
| "dpo_lambda": 0.9525156617164612, |
| "epoch": 0.9505365087673384, |
| "grad_norm": 31.56183094417888, |
| "learning_rate": 3.5377236299748147e-09, |
| "logits/chosen": -1.017897605895996, |
| "logits/rejected": -1.077573299407959, |
| "logps/chosen": -306.55389404296875, |
| "logps/rejected": -319.935791015625, |
| "loss": 0.5622, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.4877539575099945, |
| "rewards/margins": 0.3666626214981079, |
| "rewards/rejected": -0.8544166088104248, |
| "step": 454 |
| }, |
| { |
| "dpo_lambda": 0.9524108171463013, |
| "epoch": 0.9526302015179272, |
| "grad_norm": 37.50830191860258, |
| "learning_rate": 3.2374343405217884e-09, |
| "logits/chosen": -0.9689821600914001, |
| "logits/rejected": -1.0138612985610962, |
| "logps/chosen": -333.1699523925781, |
| "logps/rejected": -356.8009338378906, |
| "loss": 0.5582, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.24983340501785278, |
| "rewards/margins": 0.7474940419197083, |
| "rewards/rejected": -0.997327446937561, |
| "step": 455 |
| }, |
| { |
| "dpo_lambda": 0.9523061513900757, |
| "epoch": 0.9547238942685161, |
| "grad_norm": 35.54479092391772, |
| "learning_rate": 2.9503781785795713e-09, |
| "logits/chosen": -0.9847046732902527, |
| "logits/rejected": -1.0381470918655396, |
| "logps/chosen": -319.5542297363281, |
| "logps/rejected": -387.990234375, |
| "loss": 0.5381, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.40106528997421265, |
| "rewards/margins": 0.5943878293037415, |
| "rewards/rejected": -0.9954531192779541, |
| "step": 456 |
| }, |
| { |
| "dpo_lambda": 0.9522013068199158, |
| "epoch": 0.9568175870191049, |
| "grad_norm": 27.64991824437417, |
| "learning_rate": 2.6765705380989432e-09, |
| "logits/chosen": -0.9121294617652893, |
| "logits/rejected": -0.8870172500610352, |
| "logps/chosen": -301.5426940917969, |
| "logps/rejected": -367.2798767089844, |
| "loss": 0.5532, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.3733138144016266, |
| "rewards/margins": 0.5066189765930176, |
| "rewards/rejected": -0.8799328804016113, |
| "step": 457 |
| }, |
| { |
| "dpo_lambda": 0.9520964026451111, |
| "epoch": 0.9589112797696938, |
| "grad_norm": 36.24041769583764, |
| "learning_rate": 2.416026102552732e-09, |
| "logits/chosen": -0.9376819133758545, |
| "logits/rejected": -0.9311866760253906, |
| "logps/chosen": -327.5171203613281, |
| "logps/rejected": -355.0273742675781, |
| "loss": 0.6179, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.3835518956184387, |
| "rewards/margins": 0.282795786857605, |
| "rewards/rejected": -0.6663477420806885, |
| "step": 458 |
| }, |
| { |
| "dpo_lambda": 0.951991617679596, |
| "epoch": 0.9610049725202826, |
| "grad_norm": 27.119630162374623, |
| "learning_rate": 2.168758844148272e-09, |
| "logits/chosen": -1.0208796262741089, |
| "logits/rejected": -1.0136921405792236, |
| "logps/chosen": -353.8718566894531, |
| "logps/rejected": -364.9420471191406, |
| "loss": 0.5599, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.4231835603713989, |
| "rewards/margins": 0.5398699045181274, |
| "rewards/rejected": -0.9630534648895264, |
| "step": 459 |
| }, |
| { |
| "dpo_lambda": 0.9518867135047913, |
| "epoch": 0.9630986652708715, |
| "grad_norm": 38.794159915138245, |
| "learning_rate": 1.9347820230782295e-09, |
| "logits/chosen": -0.9907146692276001, |
| "logits/rejected": -1.038648009300232, |
| "logps/chosen": -281.57666015625, |
| "logps/rejected": -395.8458557128906, |
| "loss": 0.4965, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -0.20416496694087982, |
| "rewards/margins": 0.9015572667121887, |
| "rewards/rejected": -1.105722188949585, |
| "step": 460 |
| }, |
| { |
| "dpo_lambda": 0.9517821073532104, |
| "epoch": 0.9651923580214603, |
| "grad_norm": 18.734944359406175, |
| "learning_rate": 1.7141081868094209e-09, |
| "logits/chosen": -0.92900550365448, |
| "logits/rejected": -0.9331774115562439, |
| "logps/chosen": -385.0363464355469, |
| "logps/rejected": -431.5898742675781, |
| "loss": 0.4995, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.3005678057670593, |
| "rewards/margins": 0.7972238063812256, |
| "rewards/rejected": -1.0977916717529297, |
| "step": 461 |
| }, |
| { |
| "dpo_lambda": 0.9516772031784058, |
| "epoch": 0.9672860507720492, |
| "grad_norm": 19.901695883142253, |
| "learning_rate": 1.5067491694100153e-09, |
| "logits/chosen": -0.9556981325149536, |
| "logits/rejected": -0.9943455457687378, |
| "logps/chosen": -376.994873046875, |
| "logps/rejected": -431.7843322753906, |
| "loss": 0.5344, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.35393840074539185, |
| "rewards/margins": 0.6615272164344788, |
| "rewards/rejected": -1.0154657363891602, |
| "step": 462 |
| }, |
| { |
| "dpo_lambda": 0.9515723586082458, |
| "epoch": 0.969379743522638, |
| "grad_norm": 29.033019965362797, |
| "learning_rate": 1.3127160909147672e-09, |
| "logits/chosen": -0.9761213064193726, |
| "logits/rejected": -0.9421603679656982, |
| "logps/chosen": -364.5227966308594, |
| "logps/rejected": -408.348388671875, |
| "loss": 0.5222, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.35811877250671387, |
| "rewards/margins": 0.5714206099510193, |
| "rewards/rejected": -0.9295394420623779, |
| "step": 463 |
| }, |
| { |
| "dpo_lambda": 0.9514675140380859, |
| "epoch": 0.9714734362732269, |
| "grad_norm": 26.959809231046826, |
| "learning_rate": 1.1320193567288527e-09, |
| "logits/chosen": -0.9518054723739624, |
| "logits/rejected": -0.9945077896118164, |
| "logps/chosen": -261.4985046386719, |
| "logps/rejected": -289.08148193359375, |
| "loss": 0.5476, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.28891125321388245, |
| "rewards/margins": 0.48678356409072876, |
| "rewards/rejected": -0.7756948471069336, |
| "step": 464 |
| }, |
| { |
| "dpo_lambda": 0.951362669467926, |
| "epoch": 0.9735671290238157, |
| "grad_norm": 22.208879671906168, |
| "learning_rate": 9.64668657069706e-10, |
| "logits/chosen": -1.0004689693450928, |
| "logits/rejected": -0.997818648815155, |
| "logps/chosen": -292.04632568359375, |
| "logps/rejected": -323.6458435058594, |
| "loss": 0.5204, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.4061770439147949, |
| "rewards/margins": 0.5217178463935852, |
| "rewards/rejected": -0.9278948903083801, |
| "step": 465 |
| }, |
| { |
| "dpo_lambda": 0.9512577652931213, |
| "epoch": 0.9756608217744046, |
| "grad_norm": 18.508622786366722, |
| "learning_rate": 8.106729664475176e-10, |
| "logits/chosen": -0.9931919574737549, |
| "logits/rejected": -1.0254158973693848, |
| "logps/chosen": -320.24200439453125, |
| "logps/rejected": -325.3582458496094, |
| "loss": 0.5428, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.28171780705451965, |
| "rewards/margins": 0.5396276712417603, |
| "rewards/rejected": -0.8213454484939575, |
| "step": 466 |
| }, |
| { |
| "dpo_lambda": 0.9511529207229614, |
| "epoch": 0.9777545145249935, |
| "grad_norm": 55.96110779105031, |
| "learning_rate": 6.700405431837585e-10, |
| "logits/chosen": -0.9660060405731201, |
| "logits/rejected": -0.9795838594436646, |
| "logps/chosen": -325.99249267578125, |
| "logps/rejected": -389.2981872558594, |
| "loss": 0.49, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.21306493878364563, |
| "rewards/margins": 0.7328426241874695, |
| "rewards/rejected": -0.9459075927734375, |
| "step": 467 |
| }, |
| { |
| "dpo_lambda": 0.9510482549667358, |
| "epoch": 0.9798482072755823, |
| "grad_norm": 52.446456162690176, |
| "learning_rate": 5.427789289685347e-10, |
| "logits/chosen": -0.9498654007911682, |
| "logits/rejected": -1.0212403535842896, |
| "logps/chosen": -371.9898681640625, |
| "logps/rejected": -436.3253479003906, |
| "loss": 0.5287, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.32743874192237854, |
| "rewards/margins": 0.6464335322380066, |
| "rewards/rejected": -0.9738723039627075, |
| "step": 468 |
| }, |
| { |
| "dpo_lambda": 0.9509434103965759, |
| "epoch": 0.9819419000261712, |
| "grad_norm": 45.04906672619121, |
| "learning_rate": 4.288949484559934e-10, |
| "logits/chosen": -0.9477793574333191, |
| "logits/rejected": -0.9834456443786621, |
| "logps/chosen": -323.77288818359375, |
| "logps/rejected": -345.29022216796875, |
| "loss": 0.5288, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.3528931140899658, |
| "rewards/margins": 0.5689555406570435, |
| "rewards/rejected": -0.921848714351654, |
| "step": 469 |
| }, |
| { |
| "dpo_lambda": 0.950838565826416, |
| "epoch": 0.98403559277676, |
| "grad_norm": 25.992732802981653, |
| "learning_rate": 3.2839470889836627e-10, |
| "logits/chosen": -0.9501466155052185, |
| "logits/rejected": -0.9828510880470276, |
| "logps/chosen": -316.51806640625, |
| "logps/rejected": -342.9505615234375, |
| "loss": 0.5701, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.3308340907096863, |
| "rewards/margins": 0.5020270347595215, |
| "rewards/rejected": -0.8328611850738525, |
| "step": 470 |
| }, |
| { |
| "dpo_lambda": 0.9507337212562561, |
| "epoch": 0.9861292855273489, |
| "grad_norm": 25.58925647010266, |
| "learning_rate": 2.412835998185092e-10, |
| "logits/chosen": -0.8885989785194397, |
| "logits/rejected": -0.9959658980369568, |
| "logps/chosen": -363.73175048828125, |
| "logps/rejected": -365.086181640625, |
| "loss": 0.5018, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.3134402632713318, |
| "rewards/margins": 0.6511830687522888, |
| "rewards/rejected": -0.9646233320236206, |
| "step": 471 |
| }, |
| { |
| "dpo_lambda": 0.9506288766860962, |
| "epoch": 0.9882229782779377, |
| "grad_norm": 23.16302027932693, |
| "learning_rate": 1.6756629272085544e-10, |
| "logits/chosen": -0.9789613485336304, |
| "logits/rejected": -0.9847142696380615, |
| "logps/chosen": -333.34613037109375, |
| "logps/rejected": -401.7735595703125, |
| "loss": 0.6441, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.46712347865104675, |
| "rewards/margins": 0.3497805893421173, |
| "rewards/rejected": -0.8169040083885193, |
| "step": 472 |
| }, |
| { |
| "dpo_lambda": 0.9505242109298706, |
| "epoch": 0.9903166710285266, |
| "grad_norm": 28.986250925054538, |
| "learning_rate": 1.072467408408384e-10, |
| "logits/chosen": -1.011475682258606, |
| "logits/rejected": -0.9854814410209656, |
| "logps/chosen": -323.3624267578125, |
| "logps/rejected": -392.7468566894531, |
| "loss": 0.5504, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.43965718150138855, |
| "rewards/margins": 0.4960758090019226, |
| "rewards/rejected": -0.9357329607009888, |
| "step": 473 |
| }, |
| { |
| "dpo_lambda": 0.9504193663597107, |
| "epoch": 0.9924103637791154, |
| "grad_norm": 29.69154282719129, |
| "learning_rate": 6.032817893297793e-11, |
| "logits/chosen": -0.9288985133171082, |
| "logits/rejected": -0.9573264122009277, |
| "logps/chosen": -349.6865234375, |
| "logps/rejected": -371.0149841308594, |
| "loss": 0.5234, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.2881234288215637, |
| "rewards/margins": 0.5843805074691772, |
| "rewards/rejected": -0.872503936290741, |
| "step": 474 |
| }, |
| { |
| "dpo_lambda": 0.950314462184906, |
| "epoch": 0.9945040565297043, |
| "grad_norm": 17.52475572003096, |
| "learning_rate": 2.6813123097352287e-11, |
| "logits/chosen": -1.0027269124984741, |
| "logits/rejected": -0.9944903254508972, |
| "logps/chosen": -321.1978454589844, |
| "logps/rejected": -340.96600341796875, |
| "loss": 0.5784, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.35715794563293457, |
| "rewards/margins": 0.4749200642108917, |
| "rewards/rejected": -0.8320780396461487, |
| "step": 475 |
| }, |
| { |
| "dpo_lambda": 0.9502096176147461, |
| "epoch": 0.9965977492802931, |
| "grad_norm": 18.527439231672, |
| "learning_rate": 6.7033706447061635e-12, |
| "logits/chosen": -0.9148775935173035, |
| "logits/rejected": -0.8974156975746155, |
| "logps/chosen": -284.0632019042969, |
| "logps/rejected": -353.8758544921875, |
| "loss": 0.5733, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.34222733974456787, |
| "rewards/margins": 0.6102782487869263, |
| "rewards/rejected": -0.9525056481361389, |
| "step": 476 |
| }, |
| { |
| "dpo_lambda": 0.9501047730445862, |
| "epoch": 0.998691442030882, |
| "grad_norm": 17.038344537628184, |
| "learning_rate": 0.0, |
| "logits/chosen": -0.9984913468360901, |
| "logits/rejected": -0.9600070714950562, |
| "logps/chosen": -301.17230224609375, |
| "logps/rejected": -387.6405029296875, |
| "loss": 0.5673, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.3698651194572449, |
| "rewards/margins": 0.6756829619407654, |
| "rewards/rejected": -1.0455480813980103, |
| "step": 477 |
| }, |
| { |
| "epoch": 0.998691442030882, |
| "step": 477, |
| "total_flos": 0.0, |
| "train_loss": 0.5879578035582537, |
| "train_runtime": 40532.5341, |
| "train_samples_per_second": 1.508, |
| "train_steps_per_second": 0.012 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 477, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
