| { | |
| "best_metric": 0.2881975471973419, | |
| "best_model_checkpoint": "./mistral/21-04-24-Weni-WeniGPT-Agents-Mistral-1.0.6-SFT-1.0.11-DPO_Experiment on DPO with other hyperparameters and best SFT model of WeniGPT-2_max_steps-366_batch_4_2024-04-21_ppid_9/checkpoint-360", | |
| "epoch": 5.853658536585366, | |
| "eval_steps": 30, | |
| "global_step": 360, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.16260162601626016, | |
| "grad_norm": 11.074726104736328, | |
| "learning_rate": 3.6363636363636366e-06, | |
| "logits/chosen": -1.7639459371566772, | |
| "logits/rejected": -1.8396574258804321, | |
| "logps/chosen": -186.35623168945312, | |
| "logps/rejected": -242.7454376220703, | |
| "loss": 0.6923, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": 0.0025298120453953743, | |
| "rewards/margins": 0.002917728153988719, | |
| "rewards/rejected": -0.000387916574254632, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.3252032520325203, | |
| "grad_norm": 13.123202323913574, | |
| "learning_rate": 4.9154929577464795e-06, | |
| "logits/chosen": -1.8020210266113281, | |
| "logits/rejected": -1.827330231666565, | |
| "logps/chosen": -179.5525665283203, | |
| "logps/rejected": -159.48426818847656, | |
| "loss": 0.6821, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.0269151721149683, | |
| "rewards/margins": 0.020867865532636642, | |
| "rewards/rejected": 0.00604730611667037, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.4878048780487805, | |
| "grad_norm": 14.401883125305176, | |
| "learning_rate": 4.774647887323944e-06, | |
| "logits/chosen": -1.7544463872909546, | |
| "logits/rejected": -1.8119304180145264, | |
| "logps/chosen": -131.93418884277344, | |
| "logps/rejected": -186.62869262695312, | |
| "loss": 0.6531, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.0737195536494255, | |
| "rewards/margins": 0.08270301669836044, | |
| "rewards/rejected": -0.008983459323644638, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.4878048780487805, | |
| "eval_logits/chosen": -1.7289739847183228, | |
| "eval_logits/rejected": -1.7717361450195312, | |
| "eval_logps/chosen": -135.4998321533203, | |
| "eval_logps/rejected": -185.3151397705078, | |
| "eval_loss": 0.6393889784812927, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 0.12710317969322205, | |
| "eval_rewards/margins": 0.1346290558576584, | |
| "eval_rewards/rejected": -0.007525864988565445, | |
| "eval_runtime": 17.3728, | |
| "eval_samples_per_second": 1.612, | |
| "eval_steps_per_second": 0.806, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.6504065040650406, | |
| "grad_norm": 15.026944160461426, | |
| "learning_rate": 4.647887323943662e-06, | |
| "logits/chosen": -1.8521251678466797, | |
| "logits/rejected": -1.9095609188079834, | |
| "logps/chosen": -167.38348388671875, | |
| "logps/rejected": -215.0012969970703, | |
| "loss": 0.6343, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.11476834118366241, | |
| "rewards/margins": 0.10640791803598404, | |
| "rewards/rejected": 0.008360415697097778, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.8130081300813008, | |
| "grad_norm": 15.622106552124023, | |
| "learning_rate": 4.507042253521127e-06, | |
| "logits/chosen": -1.7336807250976562, | |
| "logits/rejected": -1.7874727249145508, | |
| "logps/chosen": -151.20423889160156, | |
| "logps/rejected": -212.6553192138672, | |
| "loss": 0.5966, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.16852515935897827, | |
| "rewards/margins": 0.19663922488689423, | |
| "rewards/rejected": -0.0281140748411417, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.975609756097561, | |
| "grad_norm": 15.992025375366211, | |
| "learning_rate": 4.3661971830985915e-06, | |
| "logits/chosen": -1.8077377080917358, | |
| "logits/rejected": -1.8177680969238281, | |
| "logps/chosen": -166.0524139404297, | |
| "logps/rejected": -166.77749633789062, | |
| "loss": 0.5732, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.28099626302719116, | |
| "rewards/margins": 0.25574439764022827, | |
| "rewards/rejected": 0.0252518467605114, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.975609756097561, | |
| "eval_logits/chosen": -1.734244465827942, | |
| "eval_logits/rejected": -1.7778174877166748, | |
| "eval_logps/chosen": -133.12557983398438, | |
| "eval_logps/rejected": -185.54177856445312, | |
| "eval_loss": 0.5475863218307495, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 0.36452820897102356, | |
| "eval_rewards/margins": 0.39471906423568726, | |
| "eval_rewards/rejected": -0.030190791934728622, | |
| "eval_runtime": 17.4075, | |
| "eval_samples_per_second": 1.609, | |
| "eval_steps_per_second": 0.804, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 1.1382113821138211, | |
| "grad_norm": 13.867135047912598, | |
| "learning_rate": 4.225352112676057e-06, | |
| "logits/chosen": -1.8027187585830688, | |
| "logits/rejected": -1.882067084312439, | |
| "logps/chosen": -174.7197723388672, | |
| "logps/rejected": -223.87844848632812, | |
| "loss": 0.5589, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2828216254711151, | |
| "rewards/margins": 0.361534059047699, | |
| "rewards/rejected": -0.07871242612600327, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 1.3008130081300813, | |
| "grad_norm": 14.660954475402832, | |
| "learning_rate": 4.0845070422535216e-06, | |
| "logits/chosen": -1.8306058645248413, | |
| "logits/rejected": -1.8653805255889893, | |
| "logps/chosen": -170.78506469726562, | |
| "logps/rejected": -210.1864013671875, | |
| "loss": 0.4755, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.4126508831977844, | |
| "rewards/margins": 0.48294001817703247, | |
| "rewards/rejected": -0.07028912007808685, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 1.4634146341463414, | |
| "grad_norm": 15.84911060333252, | |
| "learning_rate": 3.943661971830986e-06, | |
| "logits/chosen": -1.7707087993621826, | |
| "logits/rejected": -1.81671142578125, | |
| "logps/chosen": -143.22067260742188, | |
| "logps/rejected": -205.97433471679688, | |
| "loss": 0.5252, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.4095473885536194, | |
| "rewards/margins": 0.5323301553726196, | |
| "rewards/rejected": -0.12278278172016144, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 1.4634146341463414, | |
| "eval_logits/chosen": -1.74051034450531, | |
| "eval_logits/rejected": -1.7852736711502075, | |
| "eval_logps/chosen": -130.74990844726562, | |
| "eval_logps/rejected": -186.0896759033203, | |
| "eval_loss": 0.464372456073761, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 0.6020953059196472, | |
| "eval_rewards/margins": 0.6870760321617126, | |
| "eval_rewards/rejected": -0.08498072624206543, | |
| "eval_runtime": 17.3793, | |
| "eval_samples_per_second": 1.611, | |
| "eval_steps_per_second": 0.806, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 1.6260162601626016, | |
| "grad_norm": 15.015800476074219, | |
| "learning_rate": 3.8028169014084508e-06, | |
| "logits/chosen": -1.795932412147522, | |
| "logits/rejected": -1.8505065441131592, | |
| "logps/chosen": -152.42788696289062, | |
| "logps/rejected": -204.91549682617188, | |
| "loss": 0.4682, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.4214203953742981, | |
| "rewards/margins": 0.5449811816215515, | |
| "rewards/rejected": -0.12356076389551163, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 1.7886178861788617, | |
| "grad_norm": 14.755510330200195, | |
| "learning_rate": 3.6619718309859158e-06, | |
| "logits/chosen": -1.7966296672821045, | |
| "logits/rejected": -1.860388159751892, | |
| "logps/chosen": -146.94325256347656, | |
| "logps/rejected": -258.38568115234375, | |
| "loss": 0.429, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.4777925908565521, | |
| "rewards/margins": 0.659986138343811, | |
| "rewards/rejected": -0.18219345808029175, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 1.951219512195122, | |
| "grad_norm": 14.414965629577637, | |
| "learning_rate": 3.5211267605633804e-06, | |
| "logits/chosen": -1.7987533807754517, | |
| "logits/rejected": -1.8348747491836548, | |
| "logps/chosen": -159.26161193847656, | |
| "logps/rejected": -196.20147705078125, | |
| "loss": 0.4402, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.5999616384506226, | |
| "rewards/margins": 0.764100193977356, | |
| "rewards/rejected": -0.1641385406255722, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 1.951219512195122, | |
| "eval_logits/chosen": -1.7491638660430908, | |
| "eval_logits/rejected": -1.794945478439331, | |
| "eval_logps/chosen": -128.98277282714844, | |
| "eval_logps/rejected": -186.99697875976562, | |
| "eval_loss": 0.40887242555618286, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 0.7788085341453552, | |
| "eval_rewards/margins": 0.9545186161994934, | |
| "eval_rewards/rejected": -0.17571023106575012, | |
| "eval_runtime": 17.373, | |
| "eval_samples_per_second": 1.612, | |
| "eval_steps_per_second": 0.806, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 2.113821138211382, | |
| "grad_norm": 11.437810897827148, | |
| "learning_rate": 3.3802816901408454e-06, | |
| "logits/chosen": -1.7318217754364014, | |
| "logits/rejected": -1.7831417322158813, | |
| "logps/chosen": -181.61026000976562, | |
| "logps/rejected": -181.42465209960938, | |
| "loss": 0.3371, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.925055205821991, | |
| "rewards/margins": 1.1250288486480713, | |
| "rewards/rejected": -0.19997362792491913, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 2.2764227642276422, | |
| "grad_norm": 9.344759941101074, | |
| "learning_rate": 3.2394366197183104e-06, | |
| "logits/chosen": -1.8318933248519897, | |
| "logits/rejected": -1.8962070941925049, | |
| "logps/chosen": -149.0543975830078, | |
| "logps/rejected": -251.8573455810547, | |
| "loss": 0.356, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.8429641723632812, | |
| "rewards/margins": 0.9255543947219849, | |
| "rewards/rejected": -0.08259038627147675, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 2.4390243902439024, | |
| "grad_norm": 13.787588119506836, | |
| "learning_rate": 3.0985915492957746e-06, | |
| "logits/chosen": -1.7744439840316772, | |
| "logits/rejected": -1.814591646194458, | |
| "logps/chosen": -148.4331512451172, | |
| "logps/rejected": -177.21852111816406, | |
| "loss": 0.4138, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.8420242071151733, | |
| "rewards/margins": 0.8596542477607727, | |
| "rewards/rejected": -0.017630038782954216, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 2.4390243902439024, | |
| "eval_logits/chosen": -1.7541459798812866, | |
| "eval_logits/rejected": -1.8013830184936523, | |
| "eval_logps/chosen": -127.49993133544922, | |
| "eval_logps/rejected": -188.32179260253906, | |
| "eval_loss": 0.3651679456233978, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 0.9270926713943481, | |
| "eval_rewards/margins": 1.2352840900421143, | |
| "eval_rewards/rejected": -0.30819153785705566, | |
| "eval_runtime": 17.3736, | |
| "eval_samples_per_second": 1.612, | |
| "eval_steps_per_second": 0.806, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 2.6016260162601625, | |
| "grad_norm": 6.115133762359619, | |
| "learning_rate": 2.9577464788732396e-06, | |
| "logits/chosen": -1.7456220388412476, | |
| "logits/rejected": -1.834590196609497, | |
| "logps/chosen": -118.06221008300781, | |
| "logps/rejected": -232.33853149414062, | |
| "loss": 0.325, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.0931917428970337, | |
| "rewards/margins": 1.5432584285736084, | |
| "rewards/rejected": -0.4500666558742523, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 2.7642276422764227, | |
| "grad_norm": 18.476879119873047, | |
| "learning_rate": 2.8169014084507046e-06, | |
| "logits/chosen": -1.7815649509429932, | |
| "logits/rejected": -1.8541799783706665, | |
| "logps/chosen": -177.2996826171875, | |
| "logps/rejected": -254.4600067138672, | |
| "loss": 0.3698, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.6835691928863525, | |
| "rewards/margins": 1.2215800285339355, | |
| "rewards/rejected": -0.5380108952522278, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 2.926829268292683, | |
| "grad_norm": 16.214574813842773, | |
| "learning_rate": 2.676056338028169e-06, | |
| "logits/chosen": -1.8389543294906616, | |
| "logits/rejected": -1.853319525718689, | |
| "logps/chosen": -160.57228088378906, | |
| "logps/rejected": -136.90098571777344, | |
| "loss": 0.3863, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.666906476020813, | |
| "rewards/margins": 0.767067015171051, | |
| "rewards/rejected": -0.10016050189733505, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 2.926829268292683, | |
| "eval_logits/chosen": -1.7583568096160889, | |
| "eval_logits/rejected": -1.8064650297164917, | |
| "eval_logps/chosen": -126.48298645019531, | |
| "eval_logps/rejected": -189.43382263183594, | |
| "eval_loss": 0.3402670919895172, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.0287870168685913, | |
| "eval_rewards/margins": 1.4481836557388306, | |
| "eval_rewards/rejected": -0.4193967878818512, | |
| "eval_runtime": 17.3755, | |
| "eval_samples_per_second": 1.611, | |
| "eval_steps_per_second": 0.806, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 3.089430894308943, | |
| "grad_norm": 7.2306013107299805, | |
| "learning_rate": 2.535211267605634e-06, | |
| "logits/chosen": -1.8473081588745117, | |
| "logits/rejected": -1.9084949493408203, | |
| "logps/chosen": -179.919189453125, | |
| "logps/rejected": -254.76321411132812, | |
| "loss": 0.2924, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.619147002696991, | |
| "rewards/margins": 1.209044098854065, | |
| "rewards/rejected": -0.589897096157074, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 3.252032520325203, | |
| "grad_norm": 9.260942459106445, | |
| "learning_rate": 2.3943661971830984e-06, | |
| "logits/chosen": -1.8063428401947021, | |
| "logits/rejected": -1.8332942724227905, | |
| "logps/chosen": -133.8800048828125, | |
| "logps/rejected": -181.50469970703125, | |
| "loss": 0.3248, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 1.0484669208526611, | |
| "rewards/margins": 1.2968560457229614, | |
| "rewards/rejected": -0.24838905036449432, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 3.4146341463414633, | |
| "grad_norm": 17.881528854370117, | |
| "learning_rate": 2.2535211267605635e-06, | |
| "logits/chosen": -1.8409783840179443, | |
| "logits/rejected": -1.8688993453979492, | |
| "logps/chosen": -188.42379760742188, | |
| "logps/rejected": -208.37612915039062, | |
| "loss": 0.2999, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.6589190363883972, | |
| "rewards/margins": 1.1798341274261475, | |
| "rewards/rejected": -0.5209150910377502, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 3.4146341463414633, | |
| "eval_logits/chosen": -1.762878179550171, | |
| "eval_logits/rejected": -1.8114169836044312, | |
| "eval_logps/chosen": -125.83088684082031, | |
| "eval_logps/rejected": -190.80966186523438, | |
| "eval_loss": 0.3209560513496399, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.093997597694397, | |
| "eval_rewards/margins": 1.6509748697280884, | |
| "eval_rewards/rejected": -0.5569776296615601, | |
| "eval_runtime": 17.3805, | |
| "eval_samples_per_second": 1.611, | |
| "eval_steps_per_second": 0.806, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 3.5772357723577235, | |
| "grad_norm": 6.462300777435303, | |
| "learning_rate": 2.1126760563380285e-06, | |
| "logits/chosen": -1.9442710876464844, | |
| "logits/rejected": -1.963648796081543, | |
| "logps/chosen": -184.78109741210938, | |
| "logps/rejected": -209.25723266601562, | |
| "loss": 0.3841, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.808915913105011, | |
| "rewards/margins": 1.145354986190796, | |
| "rewards/rejected": -0.3364391326904297, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 3.7398373983739837, | |
| "grad_norm": 8.52739143371582, | |
| "learning_rate": 1.971830985915493e-06, | |
| "logits/chosen": -1.7989740371704102, | |
| "logits/rejected": -1.8550697565078735, | |
| "logps/chosen": -118.730712890625, | |
| "logps/rejected": -222.0437469482422, | |
| "loss": 0.3073, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.988745391368866, | |
| "rewards/margins": 1.832834005355835, | |
| "rewards/rejected": -0.8440886735916138, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 3.902439024390244, | |
| "grad_norm": 6.697244644165039, | |
| "learning_rate": 1.8309859154929579e-06, | |
| "logits/chosen": -1.8378446102142334, | |
| "logits/rejected": -1.8886947631835938, | |
| "logps/chosen": -150.81031799316406, | |
| "logps/rejected": -189.8251495361328, | |
| "loss": 0.2311, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.1466209888458252, | |
| "rewards/margins": 1.7799263000488281, | |
| "rewards/rejected": -0.6333051919937134, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 3.902439024390244, | |
| "eval_logits/chosen": -1.7646329402923584, | |
| "eval_logits/rejected": -1.8135557174682617, | |
| "eval_logps/chosen": -125.47990417480469, | |
| "eval_logps/rejected": -191.78990173339844, | |
| "eval_loss": 0.30853360891342163, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.129095196723938, | |
| "eval_rewards/margins": 1.7840988636016846, | |
| "eval_rewards/rejected": -0.655003547668457, | |
| "eval_runtime": 17.3741, | |
| "eval_samples_per_second": 1.612, | |
| "eval_steps_per_second": 0.806, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 4.065040650406504, | |
| "grad_norm": 14.190593719482422, | |
| "learning_rate": 1.6901408450704227e-06, | |
| "logits/chosen": -1.8366336822509766, | |
| "logits/rejected": -1.8869634866714478, | |
| "logps/chosen": -172.4488525390625, | |
| "logps/rejected": -221.21475219726562, | |
| "loss": 0.3199, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.31344354152679443, | |
| "rewards/margins": 0.9199511408805847, | |
| "rewards/rejected": -0.6065076589584351, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 4.227642276422764, | |
| "grad_norm": 10.718254089355469, | |
| "learning_rate": 1.5492957746478873e-06, | |
| "logits/chosen": -1.6653333902359009, | |
| "logits/rejected": -1.769636869430542, | |
| "logps/chosen": -112.0279312133789, | |
| "logps/rejected": -213.3590850830078, | |
| "loss": 0.2095, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 1.3316088914871216, | |
| "rewards/margins": 2.201448440551758, | |
| "rewards/rejected": -0.8698397874832153, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 4.390243902439025, | |
| "grad_norm": 9.33919620513916, | |
| "learning_rate": 1.4084507042253523e-06, | |
| "logits/chosen": -1.8406511545181274, | |
| "logits/rejected": -1.914043664932251, | |
| "logps/chosen": -146.75958251953125, | |
| "logps/rejected": -264.20391845703125, | |
| "loss": 0.2992, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.8267571330070496, | |
| "rewards/margins": 1.8074918985366821, | |
| "rewards/rejected": -0.9807347059249878, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 4.390243902439025, | |
| "eval_logits/chosen": -1.7653776407241821, | |
| "eval_logits/rejected": -1.8147268295288086, | |
| "eval_logps/chosen": -125.27703857421875, | |
| "eval_logps/rejected": -192.666748046875, | |
| "eval_loss": 0.30117151141166687, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.149381399154663, | |
| "eval_rewards/margins": 1.8920701742172241, | |
| "eval_rewards/rejected": -0.7426888346672058, | |
| "eval_runtime": 17.3768, | |
| "eval_samples_per_second": 1.611, | |
| "eval_steps_per_second": 0.806, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 4.5528455284552845, | |
| "grad_norm": 11.219182968139648, | |
| "learning_rate": 1.267605633802817e-06, | |
| "logits/chosen": -1.8663532733917236, | |
| "logits/rejected": -1.8739010095596313, | |
| "logps/chosen": -163.44076538085938, | |
| "logps/rejected": -166.98275756835938, | |
| "loss": 0.2959, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 1.2275232076644897, | |
| "rewards/margins": 1.5744152069091797, | |
| "rewards/rejected": -0.34689193964004517, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 4.715447154471545, | |
| "grad_norm": 10.296913146972656, | |
| "learning_rate": 1.1267605633802817e-06, | |
| "logits/chosen": -1.8331153392791748, | |
| "logits/rejected": -1.9017302989959717, | |
| "logps/chosen": -114.03620910644531, | |
| "logps/rejected": -166.3681640625, | |
| "loss": 0.2613, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.19826340675354, | |
| "rewards/margins": 1.772547960281372, | |
| "rewards/rejected": -0.5742844343185425, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 4.878048780487805, | |
| "grad_norm": 10.452792167663574, | |
| "learning_rate": 9.859154929577465e-07, | |
| "logits/chosen": -1.886589765548706, | |
| "logits/rejected": -1.9371570348739624, | |
| "logps/chosen": -125.04266357421875, | |
| "logps/rejected": -195.27493286132812, | |
| "loss": 0.2532, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.7003965973854065, | |
| "rewards/margins": 1.6694190502166748, | |
| "rewards/rejected": -0.9690224528312683, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 4.878048780487805, | |
| "eval_logits/chosen": -1.7649518251419067, | |
| "eval_logits/rejected": -1.8146189451217651, | |
| "eval_logps/chosen": -125.16932678222656, | |
| "eval_logps/rejected": -193.2909698486328, | |
| "eval_loss": 0.29433247447013855, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.1601535081863403, | |
| "eval_rewards/margins": 1.9652624130249023, | |
| "eval_rewards/rejected": -0.805108904838562, | |
| "eval_runtime": 17.3779, | |
| "eval_samples_per_second": 1.611, | |
| "eval_steps_per_second": 0.806, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 5.040650406504065, | |
| "grad_norm": 9.703845977783203, | |
| "learning_rate": 8.450704225352114e-07, | |
| "logits/chosen": -1.7542005777359009, | |
| "logits/rejected": -1.8026597499847412, | |
| "logps/chosen": -148.87220764160156, | |
| "logps/rejected": -209.79080200195312, | |
| "loss": 0.252, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.8712331652641296, | |
| "rewards/margins": 1.8061317205429077, | |
| "rewards/rejected": -0.9348985552787781, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 5.203252032520325, | |
| "grad_norm": 14.292641639709473, | |
| "learning_rate": 7.042253521126762e-07, | |
| "logits/chosen": -1.822540521621704, | |
| "logits/rejected": -1.88517165184021, | |
| "logps/chosen": -131.47857666015625, | |
| "logps/rejected": -196.138427734375, | |
| "loss": 0.2698, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 1.1736037731170654, | |
| "rewards/margins": 1.938589334487915, | |
| "rewards/rejected": -0.7649855017662048, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 5.365853658536586, | |
| "grad_norm": 16.347881317138672, | |
| "learning_rate": 5.633802816901409e-07, | |
| "logits/chosen": -1.8692938089370728, | |
| "logits/rejected": -1.905164122581482, | |
| "logps/chosen": -141.54505920410156, | |
| "logps/rejected": -183.823974609375, | |
| "loss": 0.2564, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.0674693584442139, | |
| "rewards/margins": 1.6867936849594116, | |
| "rewards/rejected": -0.6193243265151978, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 5.365853658536586, | |
| "eval_logits/chosen": -1.7652027606964111, | |
| "eval_logits/rejected": -1.815116286277771, | |
| "eval_logps/chosen": -125.05408477783203, | |
| "eval_logps/rejected": -193.71121215820312, | |
| "eval_loss": 0.2904638946056366, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.171677589416504, | |
| "eval_rewards/margins": 2.0188117027282715, | |
| "eval_rewards/rejected": -0.8471340537071228, | |
| "eval_runtime": 17.3626, | |
| "eval_samples_per_second": 1.613, | |
| "eval_steps_per_second": 0.806, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 5.528455284552845, | |
| "grad_norm": 9.766618728637695, | |
| "learning_rate": 4.225352112676057e-07, | |
| "logits/chosen": -1.8828731775283813, | |
| "logits/rejected": -1.9291940927505493, | |
| "logps/chosen": -161.82504272460938, | |
| "logps/rejected": -218.5142822265625, | |
| "loss": 0.3073, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.7544882297515869, | |
| "rewards/margins": 1.6939102411270142, | |
| "rewards/rejected": -0.9394218325614929, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 5.691056910569106, | |
| "grad_norm": 26.840465545654297, | |
| "learning_rate": 2.8169014084507043e-07, | |
| "logits/chosen": -1.8031673431396484, | |
| "logits/rejected": -1.8501522541046143, | |
| "logps/chosen": -210.4624481201172, | |
| "logps/rejected": -259.5084533691406, | |
| "loss": 0.2812, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.8144365549087524, | |
| "rewards/margins": 1.8433208465576172, | |
| "rewards/rejected": -1.0288844108581543, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 5.853658536585366, | |
| "grad_norm": 13.054983139038086, | |
| "learning_rate": 1.4084507042253522e-07, | |
| "logits/chosen": -1.798148512840271, | |
| "logits/rejected": -1.8677221536636353, | |
| "logps/chosen": -143.9618377685547, | |
| "logps/rejected": -198.14981079101562, | |
| "loss": 0.2457, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.7002367973327637, | |
| "rewards/margins": 1.3915526866912842, | |
| "rewards/rejected": -0.691315770149231, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 5.853658536585366, | |
| "eval_logits/chosen": -1.7656012773513794, | |
| "eval_logits/rejected": -1.8155311346054077, | |
| "eval_logps/chosen": -125.00711822509766, | |
| "eval_logps/rejected": -193.93080139160156, | |
| "eval_loss": 0.2881975471973419, | |
| "eval_rewards/accuracies": 0.7857142686843872, | |
| "eval_rewards/chosen": 1.1763746738433838, | |
| "eval_rewards/margins": 2.045469045639038, | |
| "eval_rewards/rejected": -0.8690943121910095, | |
| "eval_runtime": 17.372, | |
| "eval_samples_per_second": 1.612, | |
| "eval_steps_per_second": 0.806, | |
| "step": 360 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 366, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 6, | |
| "save_steps": 90, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |