| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9915492957746479, | |
| "eval_steps": 30, | |
| "global_step": 88, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": NaN, | |
| "learning_rate": 0.0001, | |
| "logits/chosen": -3.188204765319824, | |
| "logits/rejected": -2.849832534790039, | |
| "logps/chosen": -220.16908264160156, | |
| "logps/rejected": -186.17868041992188, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": NaN, | |
| "learning_rate": 0.0001, | |
| "logits/chosen": -3.417233943939209, | |
| "logits/rejected": -3.385444164276123, | |
| "logps/chosen": -511.65631103515625, | |
| "logps/rejected": -303.28558349609375, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": NaN, | |
| "learning_rate": 0.0001, | |
| "logits/chosen": -2.9685182571411133, | |
| "logits/rejected": -2.9099667072296143, | |
| "logps/chosen": -131.90631103515625, | |
| "logps/rejected": -127.09970092773438, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 13.127852439880371, | |
| "learning_rate": 9.999645980833454e-05, | |
| "logits/chosen": -3.3438305854797363, | |
| "logits/rejected": -2.7247352600097656, | |
| "logps/chosen": -212.61917114257812, | |
| "logps/rejected": -160.17083740234375, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 9.166139602661133, | |
| "learning_rate": 9.998583973465646e-05, | |
| "logits/chosen": -3.382075786590576, | |
| "logits/rejected": -2.574824094772339, | |
| "logps/chosen": -284.8162841796875, | |
| "logps/rejected": -199.35745239257812, | |
| "loss": 0.5279, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.21959751844406128, | |
| "rewards/margins": 0.6385258436203003, | |
| "rewards/rejected": -0.8581234216690063, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 7.322300910949707, | |
| "learning_rate": 9.99681412828496e-05, | |
| "logits/chosen": -3.271949291229248, | |
| "logits/rejected": -2.513960838317871, | |
| "logps/chosen": -372.6782531738281, | |
| "logps/rejected": -174.07479858398438, | |
| "loss": 0.2913, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.6500003933906555, | |
| "rewards/margins": 2.8001651763916016, | |
| "rewards/rejected": -2.1501646041870117, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 8.41703987121582, | |
| "learning_rate": 9.99433669591504e-05, | |
| "logits/chosen": -3.0961129665374756, | |
| "logits/rejected": -2.8720593452453613, | |
| "logps/chosen": -316.4073486328125, | |
| "logps/rejected": -167.79397583007812, | |
| "loss": 0.4464, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.1310131549835205, | |
| "rewards/margins": 1.2248451709747314, | |
| "rewards/rejected": -2.355858087539673, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 3.9780449867248535, | |
| "learning_rate": 9.991152027179307e-05, | |
| "logits/chosen": -2.9302220344543457, | |
| "logits/rejected": -2.5785322189331055, | |
| "logps/chosen": -165.38961791992188, | |
| "logps/rejected": -118.81493377685547, | |
| "loss": 0.1908, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.9433582425117493, | |
| "rewards/margins": 3.344132661819458, | |
| "rewards/rejected": -2.4007744789123535, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 7.300680637359619, | |
| "learning_rate": 9.987260573051269e-05, | |
| "logits/chosen": -3.1814284324645996, | |
| "logits/rejected": -3.0421230792999268, | |
| "logps/chosen": -258.6530456542969, | |
| "logps/rejected": -209.901611328125, | |
| "loss": 0.3466, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.742733895778656, | |
| "rewards/margins": 1.9031461477279663, | |
| "rewards/rejected": -2.6458799839019775, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 7.8778839111328125, | |
| "learning_rate": 9.982662884590662e-05, | |
| "logits/chosen": -2.9834322929382324, | |
| "logits/rejected": -2.546109199523926, | |
| "logps/chosen": -357.8366394042969, | |
| "logps/rejected": -137.680908203125, | |
| "loss": 0.4102, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.5062696933746338, | |
| "rewards/margins": 4.4324493408203125, | |
| "rewards/rejected": -4.938718795776367, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 11.32342529296875, | |
| "learning_rate": 9.977359612865423e-05, | |
| "logits/chosen": -2.992344379425049, | |
| "logits/rejected": -2.4726979732513428, | |
| "logps/chosen": -248.93341064453125, | |
| "logps/rejected": -167.0366668701172, | |
| "loss": 0.5224, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.211262047290802, | |
| "rewards/margins": 4.505336284637451, | |
| "rewards/rejected": -4.294074058532715, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 9.8779878616333, | |
| "learning_rate": 9.971351508859488e-05, | |
| "logits/chosen": -3.1347732543945312, | |
| "logits/rejected": -2.930368423461914, | |
| "logps/chosen": -286.450439453125, | |
| "logps/rejected": -202.72720336914062, | |
| "loss": 0.6846, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.4156379699707031, | |
| "rewards/margins": 3.582124710083008, | |
| "rewards/rejected": -4.997762680053711, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 8.25694465637207, | |
| "learning_rate": 9.964639423366442e-05, | |
| "logits/chosen": -3.121607780456543, | |
| "logits/rejected": -2.9769201278686523, | |
| "logps/chosen": -291.4040832519531, | |
| "logps/rejected": -219.06231689453125, | |
| "loss": 0.27, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.20223967730998993, | |
| "rewards/margins": 4.1018805503845215, | |
| "rewards/rejected": -3.8996407985687256, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 3.865050792694092, | |
| "learning_rate": 9.957224306869053e-05, | |
| "logits/chosen": -3.265730381011963, | |
| "logits/rejected": -2.9264979362487793, | |
| "logps/chosen": -269.4535827636719, | |
| "logps/rejected": -205.49166870117188, | |
| "loss": 0.1639, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.9554134607315063, | |
| "rewards/margins": 5.716032028198242, | |
| "rewards/rejected": -4.760618686676025, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 3.5297553539276123, | |
| "learning_rate": 9.949107209404665e-05, | |
| "logits/chosen": -2.461251735687256, | |
| "logits/rejected": -2.371189832687378, | |
| "logps/chosen": -259.4444885253906, | |
| "logps/rejected": -134.9102783203125, | |
| "loss": 0.2108, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.03092677891254425, | |
| "rewards/margins": 3.4555163383483887, | |
| "rewards/rejected": -3.4245896339416504, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 5.433193206787109, | |
| "learning_rate": 9.940289280416508e-05, | |
| "logits/chosen": -3.2052440643310547, | |
| "logits/rejected": -2.8054556846618652, | |
| "logps/chosen": -265.2724304199219, | |
| "logps/rejected": -225.47068786621094, | |
| "loss": 0.1771, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.08068099617958069, | |
| "rewards/margins": 4.355477333068848, | |
| "rewards/rejected": -4.274796485900879, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 10.783153533935547, | |
| "learning_rate": 9.930771768590933e-05, | |
| "logits/chosen": -3.0025675296783447, | |
| "logits/rejected": -3.019012451171875, | |
| "logps/chosen": -195.4888916015625, | |
| "logps/rejected": -198.30792236328125, | |
| "loss": 0.6906, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -2.109578847885132, | |
| "rewards/margins": 0.9626373052597046, | |
| "rewards/rejected": -3.072216033935547, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 6.201537132263184, | |
| "learning_rate": 9.92055602168058e-05, | |
| "logits/chosen": -3.2320845127105713, | |
| "logits/rejected": -2.9034526348114014, | |
| "logps/chosen": -226.6787109375, | |
| "logps/rejected": -132.74095153808594, | |
| "loss": 0.2037, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.05915778875350952, | |
| "rewards/margins": 3.9392547607421875, | |
| "rewards/rejected": -3.8800971508026123, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 11.768068313598633, | |
| "learning_rate": 9.909643486313533e-05, | |
| "logits/chosen": -3.178317070007324, | |
| "logits/rejected": -3.297847270965576, | |
| "logps/chosen": -412.5939636230469, | |
| "logps/rejected": -241.15750122070312, | |
| "loss": 0.5854, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.0589046478271484, | |
| "rewards/margins": 5.891679286956787, | |
| "rewards/rejected": -6.9505839347839355, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 8.77907943725586, | |
| "learning_rate": 9.898035707788463e-05, | |
| "logits/chosen": -3.237691640853882, | |
| "logits/rejected": -3.086668014526367, | |
| "logps/chosen": -311.6419372558594, | |
| "logps/rejected": -238.47996520996094, | |
| "loss": 0.4823, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.5970950126647949, | |
| "rewards/margins": 4.7628984451293945, | |
| "rewards/rejected": -5.3599934577941895, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 3.0071136951446533, | |
| "learning_rate": 9.885734329855798e-05, | |
| "logits/chosen": -3.2764744758605957, | |
| "logits/rejected": -2.930645704269409, | |
| "logps/chosen": -345.3895568847656, | |
| "logps/rejected": -216.34646606445312, | |
| "loss": 0.0629, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.1355853080749512, | |
| "rewards/margins": 6.265766620635986, | |
| "rewards/rejected": -5.130181312561035, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 11.331860542297363, | |
| "learning_rate": 9.872741094484965e-05, | |
| "logits/chosen": -3.3079967498779297, | |
| "logits/rejected": -2.7648227214813232, | |
| "logps/chosen": -351.6337585449219, | |
| "logps/rejected": -270.9600830078125, | |
| "loss": 0.369, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.9473640322685242, | |
| "rewards/margins": 2.920064687728882, | |
| "rewards/rejected": -3.8674285411834717, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 3.934148073196411, | |
| "learning_rate": 9.859057841617709e-05, | |
| "logits/chosen": -2.9965269565582275, | |
| "logits/rejected": -2.872361183166504, | |
| "logps/chosen": -424.9866943359375, | |
| "logps/rejected": -250.78515625, | |
| "loss": 0.0707, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.7661307454109192, | |
| "rewards/margins": 8.092283248901367, | |
| "rewards/rejected": -7.326152801513672, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "grad_norm": 6.69138765335083, | |
| "learning_rate": 9.844686508907537e-05, | |
| "logits/chosen": -3.3082752227783203, | |
| "logits/rejected": -3.2877731323242188, | |
| "logps/chosen": -293.59478759765625, | |
| "logps/rejected": -179.19845581054688, | |
| "loss": 0.275, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -1.0576655864715576, | |
| "rewards/margins": 2.9845128059387207, | |
| "rewards/rejected": -4.042178153991699, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "grad_norm": 14.109583854675293, | |
| "learning_rate": 9.829629131445342e-05, | |
| "logits/chosen": -2.856363534927368, | |
| "logits/rejected": -2.133051633834839, | |
| "logps/chosen": -203.2696533203125, | |
| "logps/rejected": -100.72183990478516, | |
| "loss": 0.8885, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.2501509189605713, | |
| "rewards/margins": 1.4751089811325073, | |
| "rewards/rejected": -2.725259780883789, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "grad_norm": 11.02879810333252, | |
| "learning_rate": 9.81388784147121e-05, | |
| "logits/chosen": -3.1515085697174072, | |
| "logits/rejected": -2.761265754699707, | |
| "logps/chosen": -230.35438537597656, | |
| "logps/rejected": -280.5235290527344, | |
| "loss": 0.3744, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.2650168836116791, | |
| "rewards/margins": 4.330430507659912, | |
| "rewards/rejected": -4.065413475036621, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 5.773349761962891, | |
| "learning_rate": 9.797464868072488e-05, | |
| "logits/chosen": -3.1555724143981934, | |
| "logits/rejected": -2.7849271297454834, | |
| "logps/chosen": -204.43417358398438, | |
| "logps/rejected": -110.06712341308594, | |
| "loss": 0.2692, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.146636962890625, | |
| "rewards/margins": 2.842060089111328, | |
| "rewards/rejected": -3.988697052001953, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 5.401435375213623, | |
| "learning_rate": 9.780362536868113e-05, | |
| "logits/chosen": -2.565051317214966, | |
| "logits/rejected": -2.8752522468566895, | |
| "logps/chosen": -158.33001708984375, | |
| "logps/rejected": -139.02786254882812, | |
| "loss": 0.3653, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.32712388038635254, | |
| "rewards/margins": 2.7614259719848633, | |
| "rewards/rejected": -3.088550090789795, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "grad_norm": 14.269856452941895, | |
| "learning_rate": 9.762583269679303e-05, | |
| "logits/chosen": -2.4300293922424316, | |
| "logits/rejected": -2.7993407249450684, | |
| "logps/chosen": -233.670166015625, | |
| "logps/rejected": -229.98846435546875, | |
| "loss": 0.82, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.4308565855026245, | |
| "rewards/margins": 1.704122543334961, | |
| "rewards/rejected": -3.134979248046875, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "grad_norm": 11.34128189086914, | |
| "learning_rate": 9.744129584186598e-05, | |
| "logits/chosen": -2.8739359378814697, | |
| "logits/rejected": -3.0709590911865234, | |
| "logps/chosen": -187.5973663330078, | |
| "logps/rejected": -171.49639892578125, | |
| "loss": 0.8236, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.5466258525848389, | |
| "rewards/margins": 1.1274800300598145, | |
| "rewards/rejected": -2.674105644226074, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "eval_logits/chosen": -3.2232699394226074, | |
| "eval_logits/rejected": -2.919212818145752, | |
| "eval_logps/chosen": -186.79000854492188, | |
| "eval_logps/rejected": -195.58921813964844, | |
| "eval_loss": 0.004889195319265127, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": 1.5010696649551392, | |
| "eval_rewards/margins": 7.495745658874512, | |
| "eval_rewards/rejected": -5.994676113128662, | |
| "eval_runtime": 5.0142, | |
| "eval_samples_per_second": 1.994, | |
| "eval_steps_per_second": 0.997, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "grad_norm": 13.044546127319336, | |
| "learning_rate": 9.725004093573342e-05, | |
| "logits/chosen": -2.972625255584717, | |
| "logits/rejected": -3.1085734367370605, | |
| "logps/chosen": -234.9746856689453, | |
| "logps/rejected": -170.3094024658203, | |
| "loss": 1.0699, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.7236368656158447, | |
| "rewards/margins": 2.1611523628234863, | |
| "rewards/rejected": -3.884788990020752, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "grad_norm": 5.612191200256348, | |
| "learning_rate": 9.705209506155634e-05, | |
| "logits/chosen": -2.7585391998291016, | |
| "logits/rejected": -2.7289156913757324, | |
| "logps/chosen": -283.48614501953125, | |
| "logps/rejected": -195.35939025878906, | |
| "loss": 0.1887, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.6527029275894165, | |
| "rewards/margins": 3.757754325866699, | |
| "rewards/rejected": -4.410456657409668, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "grad_norm": 5.19436502456665, | |
| "learning_rate": 9.68474862499881e-05, | |
| "logits/chosen": -2.993574857711792, | |
| "logits/rejected": -2.428236246109009, | |
| "logps/chosen": -300.1104431152344, | |
| "logps/rejected": -282.0811462402344, | |
| "loss": 0.2097, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.1330656111240387, | |
| "rewards/margins": 3.342900276184082, | |
| "rewards/rejected": -3.475965738296509, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 7.4402265548706055, | |
| "learning_rate": 9.663624347520505e-05, | |
| "logits/chosen": -3.148404359817505, | |
| "logits/rejected": -3.0182220935821533, | |
| "logps/chosen": -304.60955810546875, | |
| "logps/rejected": -140.74435424804688, | |
| "loss": 0.3689, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.3897354006767273, | |
| "rewards/margins": 3.62821102142334, | |
| "rewards/rejected": -4.017946243286133, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "grad_norm": 6.259589195251465, | |
| "learning_rate": 9.641839665080363e-05, | |
| "logits/chosen": -2.65729022026062, | |
| "logits/rejected": -2.7510809898376465, | |
| "logps/chosen": -330.6309814453125, | |
| "logps/rejected": -256.57421875, | |
| "loss": 0.156, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.08656883984804153, | |
| "rewards/margins": 4.7406206130981445, | |
| "rewards/rejected": -4.654051780700684, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 6.500273704528809, | |
| "learning_rate": 9.619397662556435e-05, | |
| "logits/chosen": -2.995790719985962, | |
| "logits/rejected": -3.0307507514953613, | |
| "logps/chosen": -308.2181701660156, | |
| "logps/rejected": -172.92543029785156, | |
| "loss": 0.2963, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.4932462275028229, | |
| "rewards/margins": 2.582528829574585, | |
| "rewards/rejected": -3.075775146484375, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "grad_norm": 5.707283973693848, | |
| "learning_rate": 9.596301517908328e-05, | |
| "logits/chosen": -3.033268451690674, | |
| "logits/rejected": -3.0536768436431885, | |
| "logps/chosen": -245.74594116210938, | |
| "logps/rejected": -160.86160278320312, | |
| "loss": 0.202, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.2917206287384033, | |
| "rewards/margins": 4.046570777893066, | |
| "rewards/rejected": -4.338291645050049, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "grad_norm": 6.139472007751465, | |
| "learning_rate": 9.572554501727198e-05, | |
| "logits/chosen": -3.1614038944244385, | |
| "logits/rejected": -2.7908077239990234, | |
| "logps/chosen": -297.92730712890625, | |
| "logps/rejected": -190.27981567382812, | |
| "loss": 0.1892, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4447726905345917, | |
| "rewards/margins": 4.278682708740234, | |
| "rewards/rejected": -4.723455429077148, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 3.791252851486206, | |
| "learning_rate": 9.548159976772592e-05, | |
| "logits/chosen": -3.0960066318511963, | |
| "logits/rejected": -2.7181763648986816, | |
| "logps/chosen": -223.69406127929688, | |
| "logps/rejected": -123.40463256835938, | |
| "loss": 0.1746, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.49881893396377563, | |
| "rewards/margins": 3.9385018348693848, | |
| "rewards/rejected": -4.437320709228516, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 11.180024147033691, | |
| "learning_rate": 9.523121397496269e-05, | |
| "logits/chosen": -3.0495779514312744, | |
| "logits/rejected": -2.918321132659912, | |
| "logps/chosen": -219.83587646484375, | |
| "logps/rejected": -188.59054565429688, | |
| "loss": 0.4327, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.5894317626953125, | |
| "rewards/margins": 2.6391894817352295, | |
| "rewards/rejected": -3.228621244430542, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "grad_norm": 7.4187703132629395, | |
| "learning_rate": 9.497442309553016e-05, | |
| "logits/chosen": -2.807313919067383, | |
| "logits/rejected": -2.6464860439300537, | |
| "logps/chosen": -189.6230010986328, | |
| "logps/rejected": -151.01107788085938, | |
| "loss": 0.693, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -3.1926302909851074, | |
| "rewards/margins": 2.0725247859954834, | |
| "rewards/rejected": -5.26515531539917, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "grad_norm": 8.239777565002441, | |
| "learning_rate": 9.471126349298556e-05, | |
| "logits/chosen": -2.6991612911224365, | |
| "logits/rejected": -2.4860870838165283, | |
| "logps/chosen": -249.05978393554688, | |
| "logps/rejected": -261.3404541015625, | |
| "loss": 0.3519, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -2.2070960998535156, | |
| "rewards/margins": 2.7090442180633545, | |
| "rewards/rejected": -4.916140079498291, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 12.044631958007812, | |
| "learning_rate": 9.444177243274618e-05, | |
| "logits/chosen": -3.069183111190796, | |
| "logits/rejected": -3.239974021911621, | |
| "logps/chosen": -191.36724853515625, | |
| "logps/rejected": -203.04444885253906, | |
| "loss": 0.8501, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -2.2000980377197266, | |
| "rewards/margins": 3.8068671226501465, | |
| "rewards/rejected": -6.006965160369873, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "grad_norm": 10.961316108703613, | |
| "learning_rate": 9.41659880768122e-05, | |
| "logits/chosen": -3.117356538772583, | |
| "logits/rejected": -3.1276466846466064, | |
| "logps/chosen": -241.28115844726562, | |
| "logps/rejected": -217.63143920898438, | |
| "loss": 0.5688, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -1.914351463317871, | |
| "rewards/margins": 3.1862049102783203, | |
| "rewards/rejected": -5.100556373596191, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "grad_norm": 11.185661315917969, | |
| "learning_rate": 9.388394947836279e-05, | |
| "logits/chosen": -3.105802536010742, | |
| "logits/rejected": -2.782811164855957, | |
| "logps/chosen": -346.5468444824219, | |
| "logps/rejected": -281.4388732910156, | |
| "loss": 0.5225, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.3682477474212646, | |
| "rewards/margins": 1.8964588642120361, | |
| "rewards/rejected": -3.264706611633301, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 11.120816230773926, | |
| "learning_rate": 9.359569657622574e-05, | |
| "logits/chosen": -3.119126319885254, | |
| "logits/rejected": -3.2095906734466553, | |
| "logps/chosen": -529.1534423828125, | |
| "logps/rejected": -279.0389404296875, | |
| "loss": 0.3314, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.4428637623786926, | |
| "rewards/margins": 6.021182060241699, | |
| "rewards/rejected": -5.578318119049072, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "grad_norm": 8.270225524902344, | |
| "learning_rate": 9.330127018922194e-05, | |
| "logits/chosen": -2.903927803039551, | |
| "logits/rejected": -2.7225542068481445, | |
| "logps/chosen": -275.26318359375, | |
| "logps/rejected": -169.46505737304688, | |
| "loss": 0.6252, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -1.2819722890853882, | |
| "rewards/margins": 2.14062237739563, | |
| "rewards/rejected": -3.4225947856903076, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "grad_norm": 3.7271859645843506, | |
| "learning_rate": 9.300071201038503e-05, | |
| "logits/chosen": -2.98264741897583, | |
| "logits/rejected": -2.486337184906006, | |
| "logps/chosen": -257.7012023925781, | |
| "logps/rejected": -152.14598083496094, | |
| "loss": 0.1543, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.4061692953109741, | |
| "rewards/margins": 4.309848785400391, | |
| "rewards/rejected": -4.716017723083496, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 5.926673889160156, | |
| "learning_rate": 9.26940646010574e-05, | |
| "logits/chosen": -3.2901933193206787, | |
| "logits/rejected": -2.8755884170532227, | |
| "logps/chosen": -191.0556640625, | |
| "logps/rejected": -104.35969543457031, | |
| "loss": 0.1687, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.4329254627227783, | |
| "rewards/margins": 4.122668266296387, | |
| "rewards/rejected": -4.555593490600586, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": NaN, | |
| "learning_rate": 9.26940646010574e-05, | |
| "logits/chosen": -3.4009902477264404, | |
| "logits/rejected": -3.2163825035095215, | |
| "logps/chosen": -228.1378936767578, | |
| "logps/rejected": -145.276123046875, | |
| "loss": 0.3099, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.2526423931121826, | |
| "rewards/margins": 4.154683589935303, | |
| "rewards/rejected": -3.90204119682312, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "grad_norm": 6.389315128326416, | |
| "learning_rate": 9.238137138486318e-05, | |
| "logits/chosen": -3.3087496757507324, | |
| "logits/rejected": -3.0156049728393555, | |
| "logps/chosen": -367.84161376953125, | |
| "logps/rejected": -198.76544189453125, | |
| "loss": 0.1843, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.8194153308868408, | |
| "rewards/margins": 4.259294033050537, | |
| "rewards/rejected": -3.439878463745117, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 2.2384748458862305, | |
| "learning_rate": 9.206267664155907e-05, | |
| "logits/chosen": -3.412761926651001, | |
| "logits/rejected": -3.3606858253479004, | |
| "logps/chosen": -394.595947265625, | |
| "logps/rejected": -234.29969787597656, | |
| "loss": 0.0957, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.6140856742858887, | |
| "rewards/margins": 3.623807191848755, | |
| "rewards/rejected": -3.009721517562866, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 9.3274507522583, | |
| "learning_rate": 9.173802550076401e-05, | |
| "logits/chosen": -2.9212708473205566, | |
| "logits/rejected": -3.2253193855285645, | |
| "logps/chosen": -309.66741943359375, | |
| "logps/rejected": -194.5745849609375, | |
| "loss": 0.3459, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.11235330998897552, | |
| "rewards/margins": 2.6622633934020996, | |
| "rewards/rejected": -2.549910068511963, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "grad_norm": 9.517582893371582, | |
| "learning_rate": 9.140746393556854e-05, | |
| "logits/chosen": -3.4288511276245117, | |
| "logits/rejected": -3.304213285446167, | |
| "logps/chosen": -376.1202392578125, | |
| "logps/rejected": -233.58489990234375, | |
| "loss": 0.5965, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.555878758430481, | |
| "rewards/margins": 4.904500961303711, | |
| "rewards/rejected": -4.3486223220825195, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "grad_norm": 1.8991745710372925, | |
| "learning_rate": 9.107103875602459e-05, | |
| "logits/chosen": -2.924043655395508, | |
| "logits/rejected": -3.185393810272217, | |
| "logps/chosen": -341.2203063964844, | |
| "logps/rejected": -234.63758850097656, | |
| "loss": 0.0427, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3599395751953125, | |
| "rewards/margins": 3.722670555114746, | |
| "rewards/rejected": -3.3627307415008545, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "grad_norm": 8.762215614318848, | |
| "learning_rate": 9.072879760251679e-05, | |
| "logits/chosen": -3.2768988609313965, | |
| "logits/rejected": -3.0862388610839844, | |
| "logps/chosen": -224.249755859375, | |
| "logps/rejected": -180.57443237304688, | |
| "loss": 0.3701, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.7867403626441956, | |
| "rewards/margins": 2.9776391983032227, | |
| "rewards/rejected": -3.7643797397613525, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 8.915589332580566, | |
| "learning_rate": 9.038078893901634e-05, | |
| "logits/chosen": -3.2617459297180176, | |
| "logits/rejected": -3.202209711074829, | |
| "logps/chosen": -89.90280151367188, | |
| "logps/rejected": -147.65475463867188, | |
| "loss": 0.4777, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.5283554792404175, | |
| "rewards/margins": 2.442681312561035, | |
| "rewards/rejected": -2.971036911010742, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "grad_norm": 11.555606842041016, | |
| "learning_rate": 9.002706204621803e-05, | |
| "logits/chosen": -3.001934766769409, | |
| "logits/rejected": -3.068469285964966, | |
| "logps/chosen": -344.8009948730469, | |
| "logps/rejected": -205.77664184570312, | |
| "loss": 0.3874, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.012376070022583008, | |
| "rewards/margins": 2.2519307136535645, | |
| "rewards/rejected": -2.2395546436309814, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "grad_norm": 14.009621620178223, | |
| "learning_rate": 8.966766701456177e-05, | |
| "logits/chosen": -2.8611111640930176, | |
| "logits/rejected": -3.4576072692871094, | |
| "logps/chosen": -338.19970703125, | |
| "logps/rejected": -260.2259521484375, | |
| "loss": 0.7432, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.9131150841712952, | |
| "rewards/margins": 2.4023218154907227, | |
| "rewards/rejected": -3.315437078475952, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "grad_norm": 4.895538330078125, | |
| "learning_rate": 8.930265473713938e-05, | |
| "logits/chosen": -3.3844141960144043, | |
| "logits/rejected": -3.3034396171569824, | |
| "logps/chosen": -369.8033447265625, | |
| "logps/rejected": -244.80897521972656, | |
| "loss": 0.2099, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.4544011354446411, | |
| "rewards/margins": 2.975295305252075, | |
| "rewards/rejected": -2.5208940505981445, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "eval_logits/chosen": -3.299164295196533, | |
| "eval_logits/rejected": -3.039053440093994, | |
| "eval_logps/chosen": -182.6027069091797, | |
| "eval_logps/rejected": -194.1580810546875, | |
| "eval_loss": 0.0008067694725468755, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": 2.757256269454956, | |
| "eval_rewards/margins": 8.322589874267578, | |
| "eval_rewards/rejected": -5.565333366394043, | |
| "eval_runtime": 4.7427, | |
| "eval_samples_per_second": 2.108, | |
| "eval_steps_per_second": 1.054, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "grad_norm": 13.20186710357666, | |
| "learning_rate": 8.893207690248776e-05, | |
| "logits/chosen": -3.109037399291992, | |
| "logits/rejected": -3.326498508453369, | |
| "logps/chosen": -275.9548034667969, | |
| "logps/rejected": -253.3433380126953, | |
| "loss": 0.6256, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.315821647644043, | |
| "rewards/margins": 1.796335220336914, | |
| "rewards/rejected": -3.112156867980957, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "grad_norm": 2.376776933670044, | |
| "learning_rate": 8.855598598726939e-05, | |
| "logits/chosen": -3.083179473876953, | |
| "logits/rejected": -3.2115023136138916, | |
| "logps/chosen": -234.42640686035156, | |
| "logps/rejected": -150.53904724121094, | |
| "loss": 0.0802, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.07728518545627594, | |
| "rewards/margins": 4.584873199462891, | |
| "rewards/rejected": -4.507587909698486, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "grad_norm": 6.479050636291504, | |
| "learning_rate": 8.817443524884119e-05, | |
| "logits/chosen": -2.5378470420837402, | |
| "logits/rejected": -2.813725471496582, | |
| "logps/chosen": -321.4371337890625, | |
| "logps/rejected": -244.43296813964844, | |
| "loss": 0.2551, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.725128173828125, | |
| "rewards/margins": 6.416964530944824, | |
| "rewards/rejected": -5.691835880279541, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "grad_norm": 8.595841407775879, | |
| "learning_rate": 8.778747871771292e-05, | |
| "logits/chosen": -3.3628735542297363, | |
| "logits/rejected": -2.6477882862091064, | |
| "logps/chosen": -491.04852294921875, | |
| "logps/rejected": -248.0843505859375, | |
| "loss": 0.4921, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.3514632284641266, | |
| "rewards/margins": 7.0789079666137695, | |
| "rewards/rejected": -6.727444648742676, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 12.204965591430664, | |
| "learning_rate": 8.739517118989605e-05, | |
| "logits/chosen": -3.0609679222106934, | |
| "logits/rejected": -3.0792436599731445, | |
| "logps/chosen": -297.3428649902344, | |
| "logps/rejected": -190.6544647216797, | |
| "loss": 0.5732, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.3494699001312256, | |
| "rewards/margins": 2.159775733947754, | |
| "rewards/rejected": -3.5092456340789795, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "grad_norm": 8.17355728149414, | |
| "learning_rate": 8.69975682191442e-05, | |
| "logits/chosen": -3.373033046722412, | |
| "logits/rejected": -3.043926239013672, | |
| "logps/chosen": -578.4942626953125, | |
| "logps/rejected": -265.8677978515625, | |
| "loss": 0.3497, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.6642241477966309, | |
| "rewards/margins": 6.849948883056641, | |
| "rewards/rejected": -7.5141730308532715, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 6.397715091705322, | |
| "learning_rate": 8.659472610908627e-05, | |
| "logits/chosen": -2.711515426635742, | |
| "logits/rejected": -2.395803928375244, | |
| "logps/chosen": -203.78604125976562, | |
| "logps/rejected": -129.5730438232422, | |
| "loss": 0.1589, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -1.117239236831665, | |
| "rewards/margins": 4.5233540534973145, | |
| "rewards/rejected": -5.640593528747559, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "grad_norm": 0.30370500683784485, | |
| "learning_rate": 8.618670190525352e-05, | |
| "logits/chosen": -3.0820484161376953, | |
| "logits/rejected": -3.0193471908569336, | |
| "logps/chosen": -378.7370300292969, | |
| "logps/rejected": -222.33255004882812, | |
| "loss": 0.0083, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.245200514793396, | |
| "rewards/margins": 7.6438798904418945, | |
| "rewards/rejected": -7.398679733276367, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "grad_norm": 10.54161548614502, | |
| "learning_rate": 8.577355338700132e-05, | |
| "logits/chosen": -3.2389307022094727, | |
| "logits/rejected": -3.0821778774261475, | |
| "logps/chosen": -346.2233581542969, | |
| "logps/rejected": -192.50180053710938, | |
| "loss": 0.7383, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -1.195668339729309, | |
| "rewards/margins": 5.02830696105957, | |
| "rewards/rejected": -6.22397518157959, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 1.390268087387085, | |
| "learning_rate": 8.535533905932738e-05, | |
| "logits/chosen": -3.3620431423187256, | |
| "logits/rejected": -2.399602174758911, | |
| "logps/chosen": -221.27833557128906, | |
| "logps/rejected": -118.21566009521484, | |
| "loss": 0.0245, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3484119176864624, | |
| "rewards/margins": 8.018238067626953, | |
| "rewards/rejected": -7.669825553894043, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 10.928914070129395, | |
| "learning_rate": 8.493211814458673e-05, | |
| "logits/chosen": -2.9451489448547363, | |
| "logits/rejected": -3.133699893951416, | |
| "logps/chosen": -201.42678833007812, | |
| "logps/rejected": -202.67453002929688, | |
| "loss": 0.3648, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -2.156554698944092, | |
| "rewards/margins": 4.664039611816406, | |
| "rewards/rejected": -6.820594310760498, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "grad_norm": 0.7106189131736755, | |
| "learning_rate": 8.450395057410561e-05, | |
| "logits/chosen": -3.4210917949676514, | |
| "logits/rejected": -2.9528050422668457, | |
| "logps/chosen": -317.3839416503906, | |
| "logps/rejected": -170.9514617919922, | |
| "loss": 0.0172, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5772140026092529, | |
| "rewards/margins": 6.574632167816162, | |
| "rewards/rejected": -7.151845455169678, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "grad_norm": 11.109986305236816, | |
| "learning_rate": 8.407089697969457e-05, | |
| "logits/chosen": -3.1691198348999023, | |
| "logits/rejected": -2.9826912879943848, | |
| "logps/chosen": -377.9816589355469, | |
| "logps/rejected": -288.35528564453125, | |
| "loss": 0.4652, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.14444704353809357, | |
| "rewards/margins": 4.671531677246094, | |
| "rewards/rejected": -4.815978527069092, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "grad_norm": 16.743736267089844, | |
| "learning_rate": 8.363301868506264e-05, | |
| "logits/chosen": -3.06179141998291, | |
| "logits/rejected": -2.977768898010254, | |
| "logps/chosen": -336.5064392089844, | |
| "logps/rejected": -273.6090393066406, | |
| "loss": 0.9035, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.7004997730255127, | |
| "rewards/margins": 3.5526602268218994, | |
| "rewards/rejected": -5.253159999847412, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "grad_norm": 4.297181606292725, | |
| "learning_rate": 8.319037769713338e-05, | |
| "logits/chosen": -3.1873130798339844, | |
| "logits/rejected": -3.2429358959198, | |
| "logps/chosen": -295.90167236328125, | |
| "logps/rejected": -252.4221954345703, | |
| "loss": 0.0913, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.35775870084762573, | |
| "rewards/margins": 7.1074934005737305, | |
| "rewards/rejected": -7.46525239944458, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 1.5024635791778564, | |
| "learning_rate": 8.274303669726426e-05, | |
| "logits/chosen": -3.2978920936584473, | |
| "logits/rejected": -3.056913375854492, | |
| "logps/chosen": -209.08226013183594, | |
| "logps/rejected": -216.75137329101562, | |
| "loss": 0.0595, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.7533246278762817, | |
| "rewards/margins": 6.2144622802734375, | |
| "rewards/rejected": -6.96778678894043, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "grad_norm": 6.813451766967773, | |
| "learning_rate": 8.229105903237044e-05, | |
| "logits/chosen": -3.1069507598876953, | |
| "logits/rejected": -2.643913507461548, | |
| "logps/chosen": -379.9039611816406, | |
| "logps/rejected": -199.5642547607422, | |
| "loss": 0.2955, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -2.039405345916748, | |
| "rewards/margins": 6.617141246795654, | |
| "rewards/rejected": -8.656547546386719, | |
| "step": 77 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 12.086407661437988, | |
| "learning_rate": 8.183450870595441e-05, | |
| "logits/chosen": -3.06406307220459, | |
| "logits/rejected": -3.0317187309265137, | |
| "logps/chosen": -369.6795654296875, | |
| "logps/rejected": -303.2950744628906, | |
| "loss": 0.4151, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.9069956541061401, | |
| "rewards/margins": 4.3820600509643555, | |
| "rewards/rejected": -6.289055824279785, | |
| "step": 78 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "grad_norm": 3.3662607669830322, | |
| "learning_rate": 8.13734503690426e-05, | |
| "logits/chosen": -3.289384126663208, | |
| "logits/rejected": -3.1391568183898926, | |
| "logps/chosen": -345.8055114746094, | |
| "logps/rejected": -193.66909790039062, | |
| "loss": 0.1584, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.599603533744812, | |
| "rewards/margins": 6.25246524810791, | |
| "rewards/rejected": -6.852068901062012, | |
| "step": 79 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 4.315209865570068, | |
| "learning_rate": 8.090794931103026e-05, | |
| "logits/chosen": -3.228783369064331, | |
| "logits/rejected": -3.0682406425476074, | |
| "logps/chosen": -270.1308898925781, | |
| "logps/rejected": -199.65390014648438, | |
| "loss": 0.197, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -1.2517391443252563, | |
| "rewards/margins": 5.22841215133667, | |
| "rewards/rejected": -6.480151176452637, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "grad_norm": 4.3422465324401855, | |
| "learning_rate": 8.043807145043604e-05, | |
| "logits/chosen": -3.3883039951324463, | |
| "logits/rejected": -2.8610763549804688, | |
| "logps/chosen": -375.4300231933594, | |
| "logps/rejected": -212.55474853515625, | |
| "loss": 0.0891, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.205045461654663, | |
| "rewards/margins": 7.316597938537598, | |
| "rewards/rejected": -8.521642684936523, | |
| "step": 81 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "grad_norm": 14.285599708557129, | |
| "learning_rate": 7.996388332556735e-05, | |
| "logits/chosen": -3.384727954864502, | |
| "logits/rejected": -3.3384907245635986, | |
| "logps/chosen": -403.5943603515625, | |
| "logps/rejected": -331.4810791015625, | |
| "loss": 0.6671, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -1.1636757850646973, | |
| "rewards/margins": 2.9779176712036133, | |
| "rewards/rejected": -4.1415934562683105, | |
| "step": 82 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "grad_norm": 0.26110827922821045, | |
| "learning_rate": 7.94854520850981e-05, | |
| "logits/chosen": -3.1430768966674805, | |
| "logits/rejected": -3.018350601196289, | |
| "logps/chosen": -502.6695556640625, | |
| "logps/rejected": -263.3882751464844, | |
| "loss": 0.0062, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.0688759088516235, | |
| "rewards/margins": 9.059045791625977, | |
| "rewards/rejected": -7.990170001983643, | |
| "step": 83 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "grad_norm": 8.293683052062988, | |
| "learning_rate": 7.900284547855991e-05, | |
| "logits/chosen": -3.26918363571167, | |
| "logits/rejected": -2.9317312240600586, | |
| "logps/chosen": -287.8534851074219, | |
| "logps/rejected": -225.27447509765625, | |
| "loss": 0.1985, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.16245371103286743, | |
| "rewards/margins": 6.57786750793457, | |
| "rewards/rejected": -6.415413856506348, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 1.314470648765564, | |
| "learning_rate": 7.85161318467482e-05, | |
| "logits/chosen": -3.219449043273926, | |
| "logits/rejected": -2.8287854194641113, | |
| "logps/chosen": -366.5806884765625, | |
| "logps/rejected": -191.7723388671875, | |
| "loss": 0.0161, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.41258037090301514, | |
| "rewards/margins": 6.966887474060059, | |
| "rewards/rejected": -6.554306983947754, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 20.908584594726562, | |
| "learning_rate": 7.80253801120447e-05, | |
| "logits/chosen": -3.160271406173706, | |
| "logits/rejected": -3.0260157585144043, | |
| "logps/chosen": -382.6925048828125, | |
| "logps/rejected": -233.9979248046875, | |
| "loss": 0.7179, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.4525451958179474, | |
| "rewards/margins": 2.8419482707977295, | |
| "rewards/rejected": -3.2944936752319336, | |
| "step": 86 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "grad_norm": 1.2079492807388306, | |
| "learning_rate": 7.753065976865744e-05, | |
| "logits/chosen": -3.191283702850342, | |
| "logits/rejected": -3.0339455604553223, | |
| "logps/chosen": -285.8114013671875, | |
| "logps/rejected": -169.56741333007812, | |
| "loss": 0.0272, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.33831024169921875, | |
| "rewards/margins": 6.575672149658203, | |
| "rewards/rejected": -6.237361907958984, | |
| "step": 87 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 0.654447615146637, | |
| "learning_rate": 7.703204087277988e-05, | |
| "logits/chosen": -3.2184348106384277, | |
| "logits/rejected": -3.2598187923431396, | |
| "logps/chosen": -163.65548706054688, | |
| "logps/rejected": -143.7698974609375, | |
| "loss": 0.0114, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.0235941410064697, | |
| "rewards/margins": 7.350224494934082, | |
| "rewards/rejected": -5.326630592346191, | |
| "step": 88 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 264, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 1, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |