| { | |
| "best_metric": 0.8975116014480591, | |
| "best_model_checkpoint": "saves/Mistral-7B-Instruct-v0.2/lora/orpo/checkpoint-1500", | |
| "epoch": 2.997999555456768, | |
| "eval_steps": 500, | |
| "global_step": 1686, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.017781729273171815, | |
| "grad_norm": 3.822810173034668, | |
| "learning_rate": 4.9995745934141085e-06, | |
| "logits/chosen": -2.5146775245666504, | |
| "logits/rejected": -2.5178749561309814, | |
| "logps/chosen": -1.456717848777771, | |
| "logps/rejected": -2.2312474250793457, | |
| "loss": 1.542, | |
| "odds_ratio_loss": 0.853233814239502, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.1456717997789383, | |
| "rewards/margins": 0.07745292782783508, | |
| "rewards/rejected": -0.22312471270561218, | |
| "sft_loss": 1.456717848777771, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.03556345854634363, | |
| "grad_norm": 37.87593078613281, | |
| "learning_rate": 4.9982812903243405e-06, | |
| "logits/chosen": -2.5083491802215576, | |
| "logits/rejected": -2.5308327674865723, | |
| "logps/chosen": -1.7510416507720947, | |
| "logps/rejected": -1.991346001625061, | |
| "loss": 1.8501, | |
| "odds_ratio_loss": 0.9901398420333862, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.17510415613651276, | |
| "rewards/margins": 0.024030480533838272, | |
| "rewards/rejected": -0.19913461804389954, | |
| "sft_loss": 1.7510416507720947, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.05334518781951545, | |
| "grad_norm": 4.208436012268066, | |
| "learning_rate": 4.996120496405222e-06, | |
| "logits/chosen": -2.520888090133667, | |
| "logits/rejected": -2.4970860481262207, | |
| "logps/chosen": -1.4711406230926514, | |
| "logps/rejected": -2.380532741546631, | |
| "loss": 1.5428, | |
| "odds_ratio_loss": 0.7161206603050232, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.14711406826972961, | |
| "rewards/margins": 0.09093920141458511, | |
| "rewards/rejected": -0.23805327713489532, | |
| "sft_loss": 1.4711406230926514, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.07112691709268726, | |
| "grad_norm": 5.997638702392578, | |
| "learning_rate": 4.99309296196014e-06, | |
| "logits/chosen": -2.5280556678771973, | |
| "logits/rejected": -2.552898645401001, | |
| "logps/chosen": -1.3411036729812622, | |
| "logps/rejected": -1.9365333318710327, | |
| "loss": 1.4122, | |
| "odds_ratio_loss": 0.711218535900116, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.13411036133766174, | |
| "rewards/margins": 0.059542976319789886, | |
| "rewards/rejected": -0.19365334510803223, | |
| "sft_loss": 1.3411036729812622, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.08890864636585907, | |
| "grad_norm": 6.682967185974121, | |
| "learning_rate": 4.989199738255166e-06, | |
| "logits/chosen": -2.5629172325134277, | |
| "logits/rejected": -2.5831732749938965, | |
| "logps/chosen": -1.3399975299835205, | |
| "logps/rejected": -1.9187812805175781, | |
| "loss": 1.4155, | |
| "odds_ratio_loss": 0.7554818391799927, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.133999764919281, | |
| "rewards/margins": 0.05787837505340576, | |
| "rewards/rejected": -0.19187815487384796, | |
| "sft_loss": 1.3399975299835205, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.1066903756390309, | |
| "grad_norm": 3.520189046859741, | |
| "learning_rate": 4.984442177154031e-06, | |
| "logits/chosen": -2.5552244186401367, | |
| "logits/rejected": -2.5440051555633545, | |
| "logps/chosen": -1.3248951435089111, | |
| "logps/rejected": -1.8020858764648438, | |
| "loss": 1.4047, | |
| "odds_ratio_loss": 0.7983237504959106, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.13248953223228455, | |
| "rewards/margins": 0.047719065099954605, | |
| "rewards/rejected": -0.18020859360694885, | |
| "sft_loss": 1.3248951435089111, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.12447210491220272, | |
| "grad_norm": 5.902307987213135, | |
| "learning_rate": 4.978821930648704e-06, | |
| "logits/chosen": -2.6206307411193848, | |
| "logits/rejected": -2.6300582885742188, | |
| "logps/chosen": -1.4133307933807373, | |
| "logps/rejected": -1.5712898969650269, | |
| "loss": 1.5087, | |
| "odds_ratio_loss": 0.9536682963371277, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.14133310317993164, | |
| "rewards/margins": 0.01579589582979679, | |
| "rewards/rejected": -0.15712900459766388, | |
| "sft_loss": 1.4133307933807373, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.14225383418537452, | |
| "grad_norm": 6.316813945770264, | |
| "learning_rate": 4.97234095028576e-06, | |
| "logits/chosen": -2.6114909648895264, | |
| "logits/rejected": -2.6050069332122803, | |
| "logps/chosen": -1.1831285953521729, | |
| "logps/rejected": -1.6871837377548218, | |
| "loss": 1.2502, | |
| "odds_ratio_loss": 0.670314371585846, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.11831285059452057, | |
| "rewards/margins": 0.050405532121658325, | |
| "rewards/rejected": -0.1687183678150177, | |
| "sft_loss": 1.1831285953521729, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.16003556345854633, | |
| "grad_norm": 2.3047008514404297, | |
| "learning_rate": 4.965001486488743e-06, | |
| "logits/chosen": -2.5930087566375732, | |
| "logits/rejected": -2.589054584503174, | |
| "logps/chosen": -1.0415079593658447, | |
| "logps/rejected": -1.5394554138183594, | |
| "loss": 1.1006, | |
| "odds_ratio_loss": 0.5907732248306274, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.10415079444646835, | |
| "rewards/margins": 0.049794748425483704, | |
| "rewards/rejected": -0.15394553542137146, | |
| "sft_loss": 1.0415079593658447, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.17781729273171815, | |
| "grad_norm": 3.8059961795806885, | |
| "learning_rate": 4.956806087776732e-06, | |
| "logits/chosen": -2.6263413429260254, | |
| "logits/rejected": -2.6199755668640137, | |
| "logps/chosen": -1.0670406818389893, | |
| "logps/rejected": -2.051734447479248, | |
| "loss": 1.1262, | |
| "odds_ratio_loss": 0.5915437340736389, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.10670407116413116, | |
| "rewards/margins": 0.09846936166286469, | |
| "rewards/rejected": -0.20517341792583466, | |
| "sft_loss": 1.0670406818389893, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.19559902200489, | |
| "grad_norm": 13.373092651367188, | |
| "learning_rate": 4.947757599879411e-06, | |
| "logits/chosen": -2.6753718852996826, | |
| "logits/rejected": -2.694586992263794, | |
| "logps/chosen": -1.2387793064117432, | |
| "logps/rejected": -1.7975523471832275, | |
| "loss": 1.3076, | |
| "odds_ratio_loss": 0.6879797577857971, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.12387792766094208, | |
| "rewards/margins": 0.05587731674313545, | |
| "rewards/rejected": -0.17975525557994843, | |
| "sft_loss": 1.2387793064117432, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.2133807512780618, | |
| "grad_norm": 3.2554187774658203, | |
| "learning_rate": 4.937859164748931e-06, | |
| "logits/chosen": -2.7004992961883545, | |
| "logits/rejected": -2.7187867164611816, | |
| "logps/chosen": -0.9789896011352539, | |
| "logps/rejected": -1.210654854774475, | |
| "loss": 1.0426, | |
| "odds_ratio_loss": 0.6358731985092163, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.09789897501468658, | |
| "rewards/margins": 0.023166518658399582, | |
| "rewards/rejected": -0.12106549739837646, | |
| "sft_loss": 0.9789896011352539, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.23116248055123362, | |
| "grad_norm": 3.08980393409729, | |
| "learning_rate": 4.92711421946891e-06, | |
| "logits/chosen": -2.654874563217163, | |
| "logits/rejected": -2.6488099098205566, | |
| "logps/chosen": -1.0345842838287354, | |
| "logps/rejected": -1.5608960390090942, | |
| "loss": 1.0981, | |
| "odds_ratio_loss": 0.6348952054977417, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.10345842689275742, | |
| "rewards/margins": 0.052631180733442307, | |
| "rewards/rejected": -0.15608961880207062, | |
| "sft_loss": 1.0345842838287354, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.24894420982440543, | |
| "grad_norm": 2.4850211143493652, | |
| "learning_rate": 4.915526495060961e-06, | |
| "logits/chosen": -2.7222416400909424, | |
| "logits/rejected": -2.716777801513672, | |
| "logps/chosen": -0.9182409048080444, | |
| "logps/rejected": -1.3934853076934814, | |
| "loss": 0.9758, | |
| "odds_ratio_loss": 0.5755457878112793, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.09182409942150116, | |
| "rewards/margins": 0.047524429857730865, | |
| "rewards/rejected": -0.13934853672981262, | |
| "sft_loss": 0.9182409048080444, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.26672593909757725, | |
| "grad_norm": 2.675926446914673, | |
| "learning_rate": 4.903100015189153e-06, | |
| "logits/chosen": -2.693967342376709, | |
| "logits/rejected": -2.7088241577148438, | |
| "logps/chosen": -0.9448526501655579, | |
| "logps/rejected": -1.2143518924713135, | |
| "loss": 1.0096, | |
| "odds_ratio_loss": 0.6471182107925415, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.09448526054620743, | |
| "rewards/margins": 0.026949917897582054, | |
| "rewards/rejected": -0.12143518775701523, | |
| "sft_loss": 0.9448526501655579, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.28450766837074903, | |
| "grad_norm": 2.688988208770752, | |
| "learning_rate": 4.889839094762848e-06, | |
| "logits/chosen": -2.6985411643981934, | |
| "logits/rejected": -2.6904616355895996, | |
| "logps/chosen": -0.9669437408447266, | |
| "logps/rejected": -1.2702093124389648, | |
| "loss": 1.0307, | |
| "odds_ratio_loss": 0.6373674869537354, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.09669436514377594, | |
| "rewards/margins": 0.030326563864946365, | |
| "rewards/rejected": -0.1270209401845932, | |
| "sft_loss": 0.9669437408447266, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.3022893976439209, | |
| "grad_norm": 2.4917891025543213, | |
| "learning_rate": 4.875748338438416e-06, | |
| "logits/chosen": -2.7611794471740723, | |
| "logits/rejected": -2.761101722717285, | |
| "logps/chosen": -1.059195876121521, | |
| "logps/rejected": -1.361277461051941, | |
| "loss": 1.1294, | |
| "odds_ratio_loss": 0.7024275064468384, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.10591959953308105, | |
| "rewards/margins": 0.03020814061164856, | |
| "rewards/rejected": -0.13612774014472961, | |
| "sft_loss": 1.059195876121521, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.32007112691709266, | |
| "grad_norm": 1.744297981262207, | |
| "learning_rate": 4.8608326390203386e-06, | |
| "logits/chosen": -2.774660348892212, | |
| "logits/rejected": -2.7589707374572754, | |
| "logps/chosen": -0.9749459028244019, | |
| "logps/rejected": -1.2964599132537842, | |
| "loss": 1.0378, | |
| "odds_ratio_loss": 0.6283431053161621, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.09749459475278854, | |
| "rewards/margins": 0.03215140849351883, | |
| "rewards/rejected": -0.12964601814746857, | |
| "sft_loss": 0.9749459028244019, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.3378528561902645, | |
| "grad_norm": 3.190150022506714, | |
| "learning_rate": 4.845097175762251e-06, | |
| "logits/chosen": -2.794633626937866, | |
| "logits/rejected": -2.8166775703430176, | |
| "logps/chosen": -1.0003185272216797, | |
| "logps/rejected": -1.2995705604553223, | |
| "loss": 1.0657, | |
| "odds_ratio_loss": 0.6539334058761597, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.10003185272216797, | |
| "rewards/margins": 0.029925208538770676, | |
| "rewards/rejected": -0.12995705008506775, | |
| "sft_loss": 1.0003185272216797, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.3556345854634363, | |
| "grad_norm": 11.579193115234375, | |
| "learning_rate": 4.8285474125685286e-06, | |
| "logits/chosen": -2.8008620738983154, | |
| "logits/rejected": -2.8179948329925537, | |
| "logps/chosen": -0.9980157017707825, | |
| "logps/rejected": -1.1520756483078003, | |
| "loss": 1.0677, | |
| "odds_ratio_loss": 0.6971891522407532, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.09980158507823944, | |
| "rewards/margins": 0.015405992977321148, | |
| "rewards/rejected": -0.11520756781101227, | |
| "sft_loss": 0.9980157017707825, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.37341631473660813, | |
| "grad_norm": 1.7257640361785889, | |
| "learning_rate": 4.811189096097025e-06, | |
| "logits/chosen": -2.795761823654175, | |
| "logits/rejected": -2.815319776535034, | |
| "logps/chosen": -0.9524224996566772, | |
| "logps/rejected": -1.2786835432052612, | |
| "loss": 1.0165, | |
| "odds_ratio_loss": 0.6408416032791138, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.09524224698543549, | |
| "rewards/margins": 0.032626111060380936, | |
| "rewards/rejected": -0.1278683841228485, | |
| "sft_loss": 0.9524224996566772, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.39119804400978, | |
| "grad_norm": 3.106210231781006, | |
| "learning_rate": 4.793028253763633e-06, | |
| "logits/chosen": -2.8339433670043945, | |
| "logits/rejected": -2.840125799179077, | |
| "logps/chosen": -0.9467460513114929, | |
| "logps/rejected": -1.1728994846343994, | |
| "loss": 1.0181, | |
| "odds_ratio_loss": 0.7131101489067078, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.09467460960149765, | |
| "rewards/margins": 0.02261533960700035, | |
| "rewards/rejected": -0.1172899454832077, | |
| "sft_loss": 0.9467460513114929, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.40897977328295176, | |
| "grad_norm": 4.078444957733154, | |
| "learning_rate": 4.774071191649352e-06, | |
| "logits/chosen": -2.811408519744873, | |
| "logits/rejected": -2.8089513778686523, | |
| "logps/chosen": -0.9588619470596313, | |
| "logps/rejected": -1.2887790203094482, | |
| "loss": 1.019, | |
| "odds_ratio_loss": 0.6018227934837341, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.09588618576526642, | |
| "rewards/margins": 0.032991714775562286, | |
| "rewards/rejected": -0.1288779079914093, | |
| "sft_loss": 0.9588619470596313, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.4267615025561236, | |
| "grad_norm": 3.046936273574829, | |
| "learning_rate": 4.7543244923105975e-06, | |
| "logits/chosen": -2.8249118328094482, | |
| "logits/rejected": -2.851069927215576, | |
| "logps/chosen": -0.9376036524772644, | |
| "logps/rejected": -1.0390188694000244, | |
| "loss": 1.0087, | |
| "odds_ratio_loss": 0.7111011147499084, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.09376037865877151, | |
| "rewards/margins": 0.010141508653759956, | |
| "rewards/rejected": -0.10390187799930573, | |
| "sft_loss": 0.9376036524772644, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.4445432318292954, | |
| "grad_norm": 1.7991715669631958, | |
| "learning_rate": 4.733795012493506e-06, | |
| "logits/chosen": -2.8495593070983887, | |
| "logits/rejected": -2.889249086380005, | |
| "logps/chosen": -1.0128796100616455, | |
| "logps/rejected": -1.202759861946106, | |
| "loss": 1.0815, | |
| "odds_ratio_loss": 0.6860855221748352, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.10128796100616455, | |
| "rewards/margins": 0.018988018855452538, | |
| "rewards/rejected": -0.12027599662542343, | |
| "sft_loss": 1.0128796100616455, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.46232496110246724, | |
| "grad_norm": 1.7875176668167114, | |
| "learning_rate": 4.712489880753035e-06, | |
| "logits/chosen": -2.814908742904663, | |
| "logits/rejected": -2.801401138305664, | |
| "logps/chosen": -0.8547848463058472, | |
| "logps/rejected": -1.1043689250946045, | |
| "loss": 0.9165, | |
| "odds_ratio_loss": 0.6170121431350708, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08547849208116531, | |
| "rewards/margins": 0.02495841309428215, | |
| "rewards/rejected": -0.11043689399957657, | |
| "sft_loss": 0.8547848463058472, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.480106690375639, | |
| "grad_norm": 1.5513663291931152, | |
| "learning_rate": 4.690416494977673e-06, | |
| "logits/chosen": -2.804611921310425, | |
| "logits/rejected": -2.835052251815796, | |
| "logps/chosen": -0.8691908717155457, | |
| "logps/rejected": -1.2830617427825928, | |
| "loss": 0.9283, | |
| "odds_ratio_loss": 0.5908734202384949, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.08691909164190292, | |
| "rewards/margins": 0.04138708487153053, | |
| "rewards/rejected": -0.12830618023872375, | |
| "sft_loss": 0.8691908717155457, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.49788841964881086, | |
| "grad_norm": 1.2906347513198853, | |
| "learning_rate": 4.667582519820639e-06, | |
| "logits/chosen": -2.8521058559417725, | |
| "logits/rejected": -2.8685240745544434, | |
| "logps/chosen": -0.9744144678115845, | |
| "logps/rejected": -1.0906678438186646, | |
| "loss": 1.0455, | |
| "odds_ratio_loss": 0.7107566595077515, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.09744144976139069, | |
| "rewards/margins": 0.011625329963862896, | |
| "rewards/rejected": -0.10906676948070526, | |
| "sft_loss": 0.9744144678115845, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.5156701489219827, | |
| "grad_norm": 3.3145501613616943, | |
| "learning_rate": 4.643995884038443e-06, | |
| "logits/chosen": -2.861464023590088, | |
| "logits/rejected": -2.889545440673828, | |
| "logps/chosen": -0.930392861366272, | |
| "logps/rejected": -1.140458345413208, | |
| "loss": 0.9957, | |
| "odds_ratio_loss": 0.6526821255683899, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.09303927421569824, | |
| "rewards/margins": 0.021006565541028976, | |
| "rewards/rejected": -0.11404584348201752, | |
| "sft_loss": 0.930392861366272, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.5334518781951545, | |
| "grad_norm": 1.5059245824813843, | |
| "learning_rate": 4.6196647777377475e-06, | |
| "logits/chosen": -2.825852870941162, | |
| "logits/rejected": -2.8473992347717285, | |
| "logps/chosen": -0.9365053176879883, | |
| "logps/rejected": -1.1387995481491089, | |
| "loss": 1.0047, | |
| "odds_ratio_loss": 0.6818416714668274, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.09365053474903107, | |
| "rewards/margins": 0.02022942714393139, | |
| "rewards/rejected": -0.1138799637556076, | |
| "sft_loss": 0.9365053176879883, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.5512336074683263, | |
| "grad_norm": 1.2869160175323486, | |
| "learning_rate": 4.59459764953147e-06, | |
| "logits/chosen": -2.8529794216156006, | |
| "logits/rejected": -2.84968900680542, | |
| "logps/chosen": -0.9238570928573608, | |
| "logps/rejected": -1.1744383573532104, | |
| "loss": 0.9884, | |
| "odds_ratio_loss": 0.6458603739738464, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.09238572418689728, | |
| "rewards/margins": 0.025058116763830185, | |
| "rewards/rejected": -0.11744382232427597, | |
| "sft_loss": 0.9238570928573608, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.5690153367414981, | |
| "grad_norm": 2.306725025177002, | |
| "learning_rate": 4.568803203605133e-06, | |
| "logits/chosen": -2.8887312412261963, | |
| "logits/rejected": -2.8795981407165527, | |
| "logps/chosen": -0.8577371835708618, | |
| "logps/rejected": -1.1274560689926147, | |
| "loss": 0.9223, | |
| "odds_ratio_loss": 0.6451677680015564, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.08577371388673782, | |
| "rewards/margins": 0.02697189524769783, | |
| "rewards/rejected": -0.11274560540914536, | |
| "sft_loss": 0.8577371835708618, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.58679706601467, | |
| "grad_norm": 1.8801417350769043, | |
| "learning_rate": 4.542290396694462e-06, | |
| "logits/chosen": -2.875821828842163, | |
| "logits/rejected": -2.8778319358825684, | |
| "logps/chosen": -0.9018943905830383, | |
| "logps/rejected": -1.1100924015045166, | |
| "loss": 0.9709, | |
| "odds_ratio_loss": 0.6898372769355774, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09018944203853607, | |
| "rewards/margins": 0.020819801837205887, | |
| "rewards/rejected": -0.11100924015045166, | |
| "sft_loss": 0.9018943905830383, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.6045787952878418, | |
| "grad_norm": 11.704841613769531, | |
| "learning_rate": 4.515068434975298e-06, | |
| "logits/chosen": -2.8479015827178955, | |
| "logits/rejected": -2.862330675125122, | |
| "logps/chosen": -0.975223183631897, | |
| "logps/rejected": -1.3013463020324707, | |
| "loss": 1.0369, | |
| "odds_ratio_loss": 0.6164206266403198, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.0975223109126091, | |
| "rewards/margins": 0.03261232003569603, | |
| "rewards/rejected": -0.13013465702533722, | |
| "sft_loss": 0.975223183631897, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.6223605245610135, | |
| "grad_norm": 1.487052321434021, | |
| "learning_rate": 4.487146770866887e-06, | |
| "logits/chosen": -2.872251510620117, | |
| "logits/rejected": -2.896749973297119, | |
| "logps/chosen": -0.9074058532714844, | |
| "logps/rejected": -1.0781655311584473, | |
| "loss": 0.9736, | |
| "odds_ratio_loss": 0.6616368293762207, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.09074059128761292, | |
| "rewards/margins": 0.01707596145570278, | |
| "rewards/rejected": -0.10781653970479965, | |
| "sft_loss": 0.9074058532714844, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.6401422538341853, | |
| "grad_norm": 2.0436906814575195, | |
| "learning_rate": 4.458535099749666e-06, | |
| "logits/chosen": -2.8917770385742188, | |
| "logits/rejected": -2.9039359092712402, | |
| "logps/chosen": -0.9911856651306152, | |
| "logps/rejected": -1.1147364377975464, | |
| "loss": 1.0655, | |
| "odds_ratio_loss": 0.7427235245704651, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.09911856800317764, | |
| "rewards/margins": 0.012355070561170578, | |
| "rewards/rejected": -0.11147364228963852, | |
| "sft_loss": 0.9911856651306152, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.6579239831073572, | |
| "grad_norm": 3.131418228149414, | |
| "learning_rate": 4.429243356598694e-06, | |
| "logits/chosen": -2.861708641052246, | |
| "logits/rejected": -2.865683078765869, | |
| "logps/chosen": -0.9219058156013489, | |
| "logps/rejected": -1.2925573587417603, | |
| "loss": 0.9872, | |
| "odds_ratio_loss": 0.6528664231300354, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.09219057857990265, | |
| "rewards/margins": 0.03706515580415726, | |
| "rewards/rejected": -0.1292557418346405, | |
| "sft_loss": 0.9219058156013489, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.675705712380529, | |
| "grad_norm": 2.098020076751709, | |
| "learning_rate": 4.399281712533875e-06, | |
| "logits/chosen": -2.8784518241882324, | |
| "logits/rejected": -2.8920931816101074, | |
| "logps/chosen": -0.857374370098114, | |
| "logps/rejected": -1.0407134294509888, | |
| "loss": 0.9257, | |
| "odds_ratio_loss": 0.6832669377326965, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.0857374519109726, | |
| "rewards/margins": 0.018333889544010162, | |
| "rewards/rejected": -0.10407133400440216, | |
| "sft_loss": 0.857374370098114, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.6934874416537008, | |
| "grad_norm": 2.2306644916534424, | |
| "learning_rate": 4.368660571288192e-06, | |
| "logits/chosen": -2.8683114051818848, | |
| "logits/rejected": -2.8982186317443848, | |
| "logps/chosen": -0.902807891368866, | |
| "logps/rejected": -1.0363978147506714, | |
| "loss": 0.9727, | |
| "odds_ratio_loss": 0.6994079947471619, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.09028079360723495, | |
| "rewards/margins": 0.013359000906348228, | |
| "rewards/rejected": -0.10363979637622833, | |
| "sft_loss": 0.902807891368866, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.7112691709268726, | |
| "grad_norm": 1.5744704008102417, | |
| "learning_rate": 4.337390565595163e-06, | |
| "logits/chosen": -2.857945680618286, | |
| "logits/rejected": -2.8643996715545654, | |
| "logps/chosen": -0.9818795919418335, | |
| "logps/rejected": -1.0570474863052368, | |
| "loss": 1.0562, | |
| "odds_ratio_loss": 0.743161678314209, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09818795323371887, | |
| "rewards/margins": 0.007516802754253149, | |
| "rewards/rejected": -0.10570474714040756, | |
| "sft_loss": 0.9818795919418335, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.7290509002000445, | |
| "grad_norm": 2.6117560863494873, | |
| "learning_rate": 4.305482553496786e-06, | |
| "logits/chosen": -2.798802137374878, | |
| "logits/rejected": -2.823525905609131, | |
| "logps/chosen": -0.8416363596916199, | |
| "logps/rejected": -1.0808361768722534, | |
| "loss": 0.9042, | |
| "odds_ratio_loss": 0.6256502270698547, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.08416363596916199, | |
| "rewards/margins": 0.023919973522424698, | |
| "rewards/rejected": -0.10808360576629639, | |
| "sft_loss": 0.8416363596916199, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.7468326294732163, | |
| "grad_norm": 2.1309070587158203, | |
| "learning_rate": 4.272947614573244e-06, | |
| "logits/chosen": -2.8514039516448975, | |
| "logits/rejected": -2.8706977367401123, | |
| "logps/chosen": -0.935457706451416, | |
| "logps/rejected": -1.069616436958313, | |
| "loss": 1.0031, | |
| "odds_ratio_loss": 0.6768142580986023, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.09354577958583832, | |
| "rewards/margins": 0.013415870256721973, | |
| "rewards/rejected": -0.10696164518594742, | |
| "sft_loss": 0.935457706451416, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.7646143587463881, | |
| "grad_norm": 1.1241884231567383, | |
| "learning_rate": 4.23979704609569e-06, | |
| "logits/chosen": -2.858297824859619, | |
| "logits/rejected": -2.8802642822265625, | |
| "logps/chosen": -0.8784758448600769, | |
| "logps/rejected": -1.0237103700637817, | |
| "loss": 0.9426, | |
| "odds_ratio_loss": 0.641250491142273, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.08784758299589157, | |
| "rewards/margins": 0.01452344935387373, | |
| "rewards/rejected": -0.10237103700637817, | |
| "sft_loss": 0.8784758448600769, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.78239608801956, | |
| "grad_norm": 1.7693474292755127, | |
| "learning_rate": 4.206042359103435e-06, | |
| "logits/chosen": -2.8127472400665283, | |
| "logits/rejected": -2.8374340534210205, | |
| "logps/chosen": -0.933555006980896, | |
| "logps/rejected": -1.194852352142334, | |
| "loss": 0.9991, | |
| "odds_ratio_loss": 0.6553630828857422, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.09335552155971527, | |
| "rewards/margins": 0.02612972818315029, | |
| "rewards/rejected": -0.11948523670434952, | |
| "sft_loss": 0.933555006980896, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.8001778172927317, | |
| "grad_norm": 1.5750656127929688, | |
| "learning_rate": 4.17169527440691e-06, | |
| "logits/chosen": -2.851017475128174, | |
| "logits/rejected": -2.85324764251709, | |
| "logps/chosen": -0.8872865438461304, | |
| "logps/rejected": -1.0435672998428345, | |
| "loss": 0.9571, | |
| "odds_ratio_loss": 0.6983404755592346, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.0887286514043808, | |
| "rewards/margins": 0.015628065913915634, | |
| "rewards/rejected": -0.10435672104358673, | |
| "sft_loss": 0.8872865438461304, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.8179595465659035, | |
| "grad_norm": 2.6487419605255127, | |
| "learning_rate": 4.136767718517797e-06, | |
| "logits/chosen": -2.8638827800750732, | |
| "logits/rejected": -2.885394811630249, | |
| "logps/chosen": -0.8147961497306824, | |
| "logps/rejected": -1.0451968908309937, | |
| "loss": 0.8763, | |
| "odds_ratio_loss": 0.6151142120361328, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.08147960901260376, | |
| "rewards/margins": 0.023040082305669785, | |
| "rewards/rejected": -0.10451970249414444, | |
| "sft_loss": 0.8147961497306824, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.8357412758390753, | |
| "grad_norm": 8.826119422912598, | |
| "learning_rate": 4.1012718195077196e-06, | |
| "logits/chosen": -2.907290458679199, | |
| "logits/rejected": -2.9450554847717285, | |
| "logps/chosen": -0.8985522985458374, | |
| "logps/rejected": -1.0472660064697266, | |
| "loss": 0.9656, | |
| "odds_ratio_loss": 0.6701352000236511, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08985523879528046, | |
| "rewards/margins": 0.014871363528072834, | |
| "rewards/rejected": -0.10472659766674042, | |
| "sft_loss": 0.8985522985458374, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.8535230051122472, | |
| "grad_norm": 1.3586628437042236, | |
| "learning_rate": 4.065219902796953e-06, | |
| "logits/chosen": -2.864061117172241, | |
| "logits/rejected": -2.8628530502319336, | |
| "logps/chosen": -0.8511291742324829, | |
| "logps/rejected": -1.0970637798309326, | |
| "loss": 0.9158, | |
| "odds_ratio_loss": 0.646445631980896, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08511291444301605, | |
| "rewards/margins": 0.024593474343419075, | |
| "rewards/rejected": -0.10970638692378998, | |
| "sft_loss": 0.8511291742324829, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.871304734385419, | |
| "grad_norm": 1.2425549030303955, | |
| "learning_rate": 4.028624486874608e-06, | |
| "logits/chosen": -2.8780291080474854, | |
| "logits/rejected": -2.901134729385376, | |
| "logps/chosen": -0.851506233215332, | |
| "logps/rejected": -1.129828691482544, | |
| "loss": 0.9181, | |
| "odds_ratio_loss": 0.6657174825668335, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08515061438083649, | |
| "rewards/margins": 0.02783224917948246, | |
| "rewards/rejected": -0.1129828691482544, | |
| "sft_loss": 0.851506233215332, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.8890864636585908, | |
| "grad_norm": 1.8243030309677124, | |
| "learning_rate": 3.99149827895177e-06, | |
| "logits/chosen": -2.9000747203826904, | |
| "logits/rejected": -2.9058332443237305, | |
| "logps/chosen": -0.9306007623672485, | |
| "logps/rejected": -1.053107500076294, | |
| "loss": 1.0001, | |
| "odds_ratio_loss": 0.6953399777412415, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09306007623672485, | |
| "rewards/margins": 0.012250673025846481, | |
| "rewards/rejected": -0.10531075298786163, | |
| "sft_loss": 0.9306007623672485, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.8890864636585908, | |
| "eval_logits/chosen": -2.8713040351867676, | |
| "eval_logits/rejected": -2.8936164379119873, | |
| "eval_logps/chosen": -0.8689887523651123, | |
| "eval_logps/rejected": -1.1122756004333496, | |
| "eval_loss": 0.9318326711654663, | |
| "eval_odds_ratio_loss": 0.6284381151199341, | |
| "eval_rewards/accuracies": 0.5920000076293945, | |
| "eval_rewards/chosen": -0.08689887821674347, | |
| "eval_rewards/margins": 0.024328680709004402, | |
| "eval_rewards/rejected": -0.11122756451368332, | |
| "eval_runtime": 189.293, | |
| "eval_samples_per_second": 5.283, | |
| "eval_sft_loss": 0.8689887523651123, | |
| "eval_steps_per_second": 2.641, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.9068681929317626, | |
| "grad_norm": 1.853105902671814, | |
| "learning_rate": 3.953854170549114e-06, | |
| "logits/chosen": -2.8680295944213867, | |
| "logits/rejected": -2.8688206672668457, | |
| "logps/chosen": -0.9001534581184387, | |
| "logps/rejected": -1.0128238201141357, | |
| "loss": 0.9674, | |
| "odds_ratio_loss": 0.6725192070007324, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.09001535177230835, | |
| "rewards/margins": 0.011267038993537426, | |
| "rewards/rejected": -0.10128238052129745, | |
| "sft_loss": 0.9001534581184387, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.9246499222049345, | |
| "grad_norm": 1.9707605838775635, | |
| "learning_rate": 3.91570523302051e-06, | |
| "logits/chosen": -2.8937981128692627, | |
| "logits/rejected": -2.9022488594055176, | |
| "logps/chosen": -0.8274223208427429, | |
| "logps/rejected": -1.105006217956543, | |
| "loss": 0.8928, | |
| "odds_ratio_loss": 0.6533216834068298, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08274223655462265, | |
| "rewards/margins": 0.027758393436670303, | |
| "rewards/rejected": -0.11050061881542206, | |
| "sft_loss": 0.8274223208427429, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.9424316514781063, | |
| "grad_norm": 1.4241600036621094, | |
| "learning_rate": 3.8770647130141996e-06, | |
| "logits/chosen": -2.892141342163086, | |
| "logits/rejected": -2.8914198875427246, | |
| "logps/chosen": -0.875231921672821, | |
| "logps/rejected": -1.0876398086547852, | |
| "loss": 0.9415, | |
| "odds_ratio_loss": 0.662711501121521, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.08752318471670151, | |
| "rewards/margins": 0.02124079503118992, | |
| "rewards/rejected": -0.10876397788524628, | |
| "sft_loss": 0.875231921672821, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.960213380751278, | |
| "grad_norm": 2.250497341156006, | |
| "learning_rate": 3.837946027873086e-06, | |
| "logits/chosen": -2.8456499576568604, | |
| "logits/rejected": -2.8566787242889404, | |
| "logps/chosen": -0.9233220815658569, | |
| "logps/rejected": -1.1583213806152344, | |
| "loss": 0.9895, | |
| "odds_ratio_loss": 0.6621237993240356, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.09233220666646957, | |
| "rewards/margins": 0.02349993959069252, | |
| "rewards/rejected": -0.11583214998245239, | |
| "sft_loss": 0.9233220815658569, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.9779951100244498, | |
| "grad_norm": 5.65519905090332, | |
| "learning_rate": 3.7983627609757713e-06, | |
| "logits/chosen": -2.8913745880126953, | |
| "logits/rejected": -2.901867389678955, | |
| "logps/chosen": -0.9292505383491516, | |
| "logps/rejected": -1.059091329574585, | |
| "loss": 0.996, | |
| "odds_ratio_loss": 0.6679803133010864, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.0929250493645668, | |
| "rewards/margins": 0.012984082102775574, | |
| "rewards/rejected": -0.10590913146734238, | |
| "sft_loss": 0.9292505383491516, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.9957768392976217, | |
| "grad_norm": 1.4165409803390503, | |
| "learning_rate": 3.758328657019924e-06, | |
| "logits/chosen": -2.8648407459259033, | |
| "logits/rejected": -2.8767480850219727, | |
| "logps/chosen": -0.8623329997062683, | |
| "logps/rejected": -1.1223938465118408, | |
| "loss": 0.9301, | |
| "odds_ratio_loss": 0.6778011322021484, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.08623330295085907, | |
| "rewards/margins": 0.026006082072854042, | |
| "rewards/rejected": -0.11223938316106796, | |
| "sft_loss": 0.8623329997062683, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 1.0135585685707935, | |
| "grad_norm": 6.480414867401123, | |
| "learning_rate": 3.717857617249642e-06, | |
| "logits/chosen": -2.8424503803253174, | |
| "logits/rejected": -2.8727316856384277, | |
| "logps/chosen": -0.9302489161491394, | |
| "logps/rejected": -1.159745693206787, | |
| "loss": 0.9995, | |
| "odds_ratio_loss": 0.6924289464950562, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.09302489459514618, | |
| "rewards/margins": 0.02294967696070671, | |
| "rewards/rejected": -0.11597456783056259, | |
| "sft_loss": 0.9302489161491394, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 1.0313402978439654, | |
| "grad_norm": 1.2129720449447632, | |
| "learning_rate": 3.6769636946284543e-06, | |
| "logits/chosen": -2.8574068546295166, | |
| "logits/rejected": -2.8658154010772705, | |
| "logps/chosen": -0.8263272047042847, | |
| "logps/rejected": -1.041471242904663, | |
| "loss": 0.8894, | |
| "odds_ratio_loss": 0.6303838491439819, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08263272792100906, | |
| "rewards/margins": 0.02151440642774105, | |
| "rewards/rejected": -0.10414713621139526, | |
| "sft_loss": 0.8263272047042847, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 1.049122027117137, | |
| "grad_norm": 1.3920116424560547, | |
| "learning_rate": 3.6356610889596355e-06, | |
| "logits/chosen": -2.8504767417907715, | |
| "logits/rejected": -2.8750224113464355, | |
| "logps/chosen": -0.891506016254425, | |
| "logps/rejected": -1.043367624282837, | |
| "loss": 0.9578, | |
| "odds_ratio_loss": 0.6629586815834045, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08915059268474579, | |
| "rewards/margins": 0.01518617570400238, | |
| "rewards/rejected": -0.10433676093816757, | |
| "sft_loss": 0.891506016254425, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 1.066903756390309, | |
| "grad_norm": 1.6321930885314941, | |
| "learning_rate": 3.593964141955541e-06, | |
| "logits/chosen": -2.851910352706909, | |
| "logits/rejected": -2.856790065765381, | |
| "logps/chosen": -0.849289059638977, | |
| "logps/rejected": -0.9652984738349915, | |
| "loss": 0.9171, | |
| "odds_ratio_loss": 0.6785116195678711, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08492890745401382, | |
| "rewards/margins": 0.011600947007536888, | |
| "rewards/rejected": -0.09652985632419586, | |
| "sft_loss": 0.849289059638977, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 1.0846854856634809, | |
| "grad_norm": 1.705749273300171, | |
| "learning_rate": 3.5518873322576573e-06, | |
| "logits/chosen": -2.8073315620422363, | |
| "logits/rejected": -2.8466951847076416, | |
| "logps/chosen": -0.8435554504394531, | |
| "logps/rejected": -1.0204975605010986, | |
| "loss": 0.9069, | |
| "odds_ratio_loss": 0.6330701112747192, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08435554802417755, | |
| "rewards/margins": 0.017694219946861267, | |
| "rewards/rejected": -0.10204975306987762, | |
| "sft_loss": 0.8435554504394531, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 1.1024672149366526, | |
| "grad_norm": 1.2050608396530151, | |
| "learning_rate": 3.5094452704091143e-06, | |
| "logits/chosen": -2.8757572174072266, | |
| "logits/rejected": -2.8627264499664307, | |
| "logps/chosen": -0.8311964273452759, | |
| "logps/rejected": -1.010647177696228, | |
| "loss": 0.8957, | |
| "odds_ratio_loss": 0.6448286175727844, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08311963826417923, | |
| "rewards/margins": 0.01794508285820484, | |
| "rewards/rejected": -0.10106471925973892, | |
| "sft_loss": 0.8311964273452759, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 1.1202489442098245, | |
| "grad_norm": 2.8746261596679688, | |
| "learning_rate": 3.46665269378139e-06, | |
| "logits/chosen": -2.834296941757202, | |
| "logits/rejected": -2.8471195697784424, | |
| "logps/chosen": -0.8721610903739929, | |
| "logps/rejected": -1.0536631345748901, | |
| "loss": 0.9388, | |
| "odds_ratio_loss": 0.6662768125534058, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08721610903739929, | |
| "rewards/margins": 0.018150202929973602, | |
| "rewards/rejected": -0.1053663119673729, | |
| "sft_loss": 0.8721610903739929, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 1.1380306734829961, | |
| "grad_norm": 2.6889796257019043, | |
| "learning_rate": 3.4235244614569794e-06, | |
| "logits/chosen": -2.8430144786834717, | |
| "logits/rejected": -2.841323137283325, | |
| "logps/chosen": -0.9479130506515503, | |
| "logps/rejected": -1.0536433458328247, | |
| "loss": 1.0213, | |
| "odds_ratio_loss": 0.7333841919898987, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.09479130804538727, | |
| "rewards/margins": 0.010573023930191994, | |
| "rewards/rejected": -0.10536432266235352, | |
| "sft_loss": 0.9479130506515503, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 1.155812402756168, | |
| "grad_norm": 1.1239742040634155, | |
| "learning_rate": 3.3800755490698008e-06, | |
| "logits/chosen": -2.867356777191162, | |
| "logits/rejected": -2.8678269386291504, | |
| "logps/chosen": -0.8813873529434204, | |
| "logps/rejected": -1.146475076675415, | |
| "loss": 0.9416, | |
| "odds_ratio_loss": 0.6019629240036011, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08813872933387756, | |
| "rewards/margins": 0.02650878205895424, | |
| "rewards/rejected": -0.1146475300192833, | |
| "sft_loss": 0.8813873529434204, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 1.17359413202934, | |
| "grad_norm": 1.5449692010879517, | |
| "learning_rate": 3.3363210436051287e-06, | |
| "logits/chosen": -2.8728928565979004, | |
| "logits/rejected": -2.8769795894622803, | |
| "logps/chosen": -0.9058928489685059, | |
| "logps/rejected": -1.1068971157073975, | |
| "loss": 0.975, | |
| "odds_ratio_loss": 0.6907466650009155, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.09058928489685059, | |
| "rewards/margins": 0.0201004259288311, | |
| "rewards/rejected": -0.11068971455097198, | |
| "sft_loss": 0.9058928489685059, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 1.1913758613025116, | |
| "grad_norm": 1.958606243133545, | |
| "learning_rate": 3.292276138160867e-06, | |
| "logits/chosen": -2.8424530029296875, | |
| "logits/rejected": -2.854606866836548, | |
| "logps/chosen": -0.8394180536270142, | |
| "logps/rejected": -1.0364609956741333, | |
| "loss": 0.9055, | |
| "odds_ratio_loss": 0.6611912250518799, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08394180238246918, | |
| "rewards/margins": 0.019704295322299004, | |
| "rewards/rejected": -0.10364609956741333, | |
| "sft_loss": 0.8394180536270142, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 1.2091575905756835, | |
| "grad_norm": 3.3439667224884033, | |
| "learning_rate": 3.2479561266719694e-06, | |
| "logits/chosen": -2.8539648056030273, | |
| "logits/rejected": -2.8596293926239014, | |
| "logps/chosen": -0.89495450258255, | |
| "logps/rejected": -1.0723049640655518, | |
| "loss": 0.9604, | |
| "odds_ratio_loss": 0.6547910571098328, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.0894954577088356, | |
| "rewards/margins": 0.017735039815306664, | |
| "rewards/rejected": -0.10723049938678741, | |
| "sft_loss": 0.89495450258255, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 1.2269393198488552, | |
| "grad_norm": 3.391028881072998, | |
| "learning_rate": 3.2033763985998533e-06, | |
| "logits/chosen": -2.8784637451171875, | |
| "logits/rejected": -2.878801107406616, | |
| "logps/chosen": -0.8193685412406921, | |
| "logps/rejected": -1.303346037864685, | |
| "loss": 0.8782, | |
| "odds_ratio_loss": 0.5878696441650391, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.08193685114383698, | |
| "rewards/margins": 0.048397745937108994, | |
| "rewards/rejected": -0.13033458590507507, | |
| "sft_loss": 0.8193685412406921, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 1.244721049122027, | |
| "grad_norm": 1.8017568588256836, | |
| "learning_rate": 3.1585524335886335e-06, | |
| "logits/chosen": -2.8997411727905273, | |
| "logits/rejected": -2.891542911529541, | |
| "logps/chosen": -0.8149515986442566, | |
| "logps/rejected": -1.062873125076294, | |
| "loss": 0.8781, | |
| "odds_ratio_loss": 0.6314008831977844, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08149515837430954, | |
| "rewards/margins": 0.024792145937681198, | |
| "rewards/rejected": -0.10628731548786163, | |
| "sft_loss": 0.8149515986442566, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 1.262502778395199, | |
| "grad_norm": 2.5622735023498535, | |
| "learning_rate": 3.1134997960900536e-06, | |
| "logits/chosen": -2.832627296447754, | |
| "logits/rejected": -2.8471617698669434, | |
| "logps/chosen": -0.8054940104484558, | |
| "logps/rejected": -1.1920253038406372, | |
| "loss": 0.864, | |
| "odds_ratio_loss": 0.58550626039505, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.08054940402507782, | |
| "rewards/margins": 0.03865312412381172, | |
| "rewards/rejected": -0.11920253187417984, | |
| "sft_loss": 0.8054940104484558, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 1.2802845076683709, | |
| "grad_norm": 1.4196934700012207, | |
| "learning_rate": 3.0682341299589583e-06, | |
| "logits/chosen": -2.857084274291992, | |
| "logits/rejected": -2.8681209087371826, | |
| "logps/chosen": -0.8212624788284302, | |
| "logps/rejected": -0.9836801290512085, | |
| "loss": 0.887, | |
| "odds_ratio_loss": 0.6570355296134949, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08212624490261078, | |
| "rewards/margins": 0.016241777688264847, | |
| "rewards/rejected": -0.09836802631616592, | |
| "sft_loss": 0.8212624788284302, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 1.2980662369415426, | |
| "grad_norm": 1.5716747045516968, | |
| "learning_rate": 3.022771153021201e-06, | |
| "logits/chosen": -2.860788583755493, | |
| "logits/rejected": -2.8880984783172607, | |
| "logps/chosen": -0.8381561040878296, | |
| "logps/rejected": -1.052847981452942, | |
| "loss": 0.9036, | |
| "odds_ratio_loss": 0.6544219255447388, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08381561934947968, | |
| "rewards/margins": 0.021469179540872574, | |
| "rewards/rejected": -0.10528478771448135, | |
| "sft_loss": 0.8381561040878296, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 1.3158479662147144, | |
| "grad_norm": 1.4270474910736084, | |
| "learning_rate": 2.9771266516158625e-06, | |
| "logits/chosen": -2.8542871475219727, | |
| "logits/rejected": -2.877516508102417, | |
| "logps/chosen": -0.8428620100021362, | |
| "logps/rejected": -1.029301404953003, | |
| "loss": 0.9101, | |
| "odds_ratio_loss": 0.6720696687698364, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08428619801998138, | |
| "rewards/margins": 0.0186439398676157, | |
| "rewards/rejected": -0.10293014347553253, | |
| "sft_loss": 0.8428620100021362, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 1.3336296954878861, | |
| "grad_norm": 1.401987910270691, | |
| "learning_rate": 2.9313164751136802e-06, | |
| "logits/chosen": -2.840547561645508, | |
| "logits/rejected": -2.872194766998291, | |
| "logps/chosen": -0.8261799812316895, | |
| "logps/rejected": -1.0598170757293701, | |
| "loss": 0.8865, | |
| "odds_ratio_loss": 0.6030489802360535, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.08261799812316895, | |
| "rewards/margins": 0.023363707587122917, | |
| "rewards/rejected": -0.1059817224740982, | |
| "sft_loss": 0.8261799812316895, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 1.351411424761058, | |
| "grad_norm": 2.474881410598755, | |
| "learning_rate": 2.8853565304135956e-06, | |
| "logits/chosen": -2.8949756622314453, | |
| "logits/rejected": -2.898646593093872, | |
| "logps/chosen": -0.918350338935852, | |
| "logps/rejected": -1.0580472946166992, | |
| "loss": 0.9899, | |
| "odds_ratio_loss": 0.7158701419830322, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.09183504432439804, | |
| "rewards/margins": 0.013969694264233112, | |
| "rewards/rejected": -0.10580474138259888, | |
| "sft_loss": 0.918350338935852, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 1.36919315403423, | |
| "grad_norm": 2.682598829269409, | |
| "learning_rate": 2.839262776419313e-06, | |
| "logits/chosen": -2.8501102924346924, | |
| "logits/rejected": -2.85284686088562, | |
| "logps/chosen": -0.8370243906974792, | |
| "logps/rejected": -1.3541486263275146, | |
| "loss": 0.8958, | |
| "odds_ratio_loss": 0.5878289937973022, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.0837024375796318, | |
| "rewards/margins": 0.05171241611242294, | |
| "rewards/rejected": -0.13541486859321594, | |
| "sft_loss": 0.8370243906974792, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 1.3869748833074016, | |
| "grad_norm": 2.036529064178467, | |
| "learning_rate": 2.793051218497817e-06, | |
| "logits/chosen": -2.871901035308838, | |
| "logits/rejected": -2.880750894546509, | |
| "logps/chosen": -0.8279402852058411, | |
| "logps/rejected": -0.9436777830123901, | |
| "loss": 0.8955, | |
| "odds_ratio_loss": 0.6755216717720032, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08279402554035187, | |
| "rewards/margins": 0.011573752388358116, | |
| "rewards/rejected": -0.09436777979135513, | |
| "sft_loss": 0.8279402852058411, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 1.4047566125805735, | |
| "grad_norm": 1.4822628498077393, | |
| "learning_rate": 2.7467379029217437e-06, | |
| "logits/chosen": -2.848276138305664, | |
| "logits/rejected": -2.8498966693878174, | |
| "logps/chosen": -0.8329153060913086, | |
| "logps/rejected": -1.1663061380386353, | |
| "loss": 0.8957, | |
| "odds_ratio_loss": 0.6282148957252502, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08329152315855026, | |
| "rewards/margins": 0.033339083194732666, | |
| "rewards/rejected": -0.11663061380386353, | |
| "sft_loss": 0.8329153060913086, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 1.4225383418537452, | |
| "grad_norm": 1.2180672883987427, | |
| "learning_rate": 2.7003389112975546e-06, | |
| "logits/chosen": -2.8609681129455566, | |
| "logits/rejected": -2.8844683170318604, | |
| "logps/chosen": -0.8761266469955444, | |
| "logps/rejected": -1.083616018295288, | |
| "loss": 0.9409, | |
| "odds_ratio_loss": 0.6474456787109375, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08761266618967056, | |
| "rewards/margins": 0.020748943090438843, | |
| "rewards/rejected": -0.10836161673069, | |
| "sft_loss": 0.8761266469955444, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 1.440320071126917, | |
| "grad_norm": 3.231017589569092, | |
| "learning_rate": 2.653870354981437e-06, | |
| "logits/chosen": -2.858649969100952, | |
| "logits/rejected": -2.870346784591675, | |
| "logps/chosen": -0.7923532128334045, | |
| "logps/rejected": -1.0410521030426025, | |
| "loss": 0.8559, | |
| "odds_ratio_loss": 0.6351376175880432, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.07923531532287598, | |
| "rewards/margins": 0.024869883432984352, | |
| "rewards/rejected": -0.10410521179437637, | |
| "sft_loss": 0.7923532128334045, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 1.458101800400089, | |
| "grad_norm": 4.5185065269470215, | |
| "learning_rate": 2.6073483694848777e-06, | |
| "logits/chosen": -2.827204465866089, | |
| "logits/rejected": -2.8571865558624268, | |
| "logps/chosen": -0.8361181020736694, | |
| "logps/rejected": -1.0542662143707275, | |
| "loss": 0.9018, | |
| "odds_ratio_loss": 0.6572374701499939, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08361181616783142, | |
| "rewards/margins": 0.02181481197476387, | |
| "rewards/rejected": -0.10542663186788559, | |
| "sft_loss": 0.8361181020736694, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 1.4758835296732609, | |
| "grad_norm": 1.3891066312789917, | |
| "learning_rate": 2.560789108871847e-06, | |
| "logits/chosen": -2.8242173194885254, | |
| "logits/rejected": -2.8416523933410645, | |
| "logps/chosen": -0.8545220494270325, | |
| "logps/rejected": -1.1704602241516113, | |
| "loss": 0.9191, | |
| "odds_ratio_loss": 0.6462253928184509, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.08545219898223877, | |
| "rewards/margins": 0.03159382566809654, | |
| "rewards/rejected": -0.11704603582620621, | |
| "sft_loss": 0.8545220494270325, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 1.4936652589464325, | |
| "grad_norm": 3.542637348175049, | |
| "learning_rate": 2.514208740149544e-06, | |
| "logits/chosen": -2.8630993366241455, | |
| "logits/rejected": -2.892916440963745, | |
| "logps/chosen": -0.8821213841438293, | |
| "logps/rejected": -1.1250375509262085, | |
| "loss": 0.9496, | |
| "odds_ratio_loss": 0.6751636862754822, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08821214735507965, | |
| "rewards/margins": 0.024291623383760452, | |
| "rewards/rejected": -0.11250375211238861, | |
| "sft_loss": 0.8821213841438293, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 1.5114469882196042, | |
| "grad_norm": 1.9266990423202515, | |
| "learning_rate": 2.46762343765464e-06, | |
| "logits/chosen": -2.8725075721740723, | |
| "logits/rejected": -2.877906084060669, | |
| "logps/chosen": -0.8815126419067383, | |
| "logps/rejected": -1.12534499168396, | |
| "loss": 0.9432, | |
| "odds_ratio_loss": 0.6172918081283569, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08815126121044159, | |
| "rewards/margins": 0.02438322827219963, | |
| "rewards/rejected": -0.11253450065851212, | |
| "sft_loss": 0.8815126419067383, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 1.5292287174927761, | |
| "grad_norm": 1.9089457988739014, | |
| "learning_rate": 2.4210493774369903e-06, | |
| "logits/chosen": -2.843306064605713, | |
| "logits/rejected": -2.844432830810547, | |
| "logps/chosen": -0.8710628747940063, | |
| "logps/rejected": -1.0414961576461792, | |
| "loss": 0.9387, | |
| "odds_ratio_loss": 0.6761898398399353, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08710628747940063, | |
| "rewards/margins": 0.01704334281384945, | |
| "rewards/rejected": -0.10414961725473404, | |
| "sft_loss": 0.8710628747940063, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 1.547010446765948, | |
| "grad_norm": 1.2754076719284058, | |
| "learning_rate": 2.374502731642732e-06, | |
| "logits/chosen": -2.855776309967041, | |
| "logits/rejected": -2.8679866790771484, | |
| "logps/chosen": -0.926120400428772, | |
| "logps/rejected": -1.1001455783843994, | |
| "loss": 0.9908, | |
| "odds_ratio_loss": 0.6463494896888733, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.09261203557252884, | |
| "rewards/margins": 0.01740253157913685, | |
| "rewards/rejected": -0.11001457273960114, | |
| "sft_loss": 0.926120400428772, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 1.56479217603912, | |
| "grad_norm": 1.6846286058425903, | |
| "learning_rate": 2.3279996628987556e-06, | |
| "logits/chosen": -2.844256639480591, | |
| "logits/rejected": -2.8760454654693604, | |
| "logps/chosen": -0.8887776136398315, | |
| "logps/rejected": -1.0579562187194824, | |
| "loss": 0.9585, | |
| "odds_ratio_loss": 0.6975380778312683, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08887775987386703, | |
| "rewards/margins": 0.01691785827279091, | |
| "rewards/rejected": -0.10579562187194824, | |
| "sft_loss": 0.8887776136398315, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 1.5825739053122916, | |
| "grad_norm": 1.5345253944396973, | |
| "learning_rate": 2.281556318700474e-06, | |
| "logits/chosen": -2.851228952407837, | |
| "logits/rejected": -2.8857483863830566, | |
| "logps/chosen": -0.82933509349823, | |
| "logps/rejected": -0.9570034742355347, | |
| "loss": 0.8991, | |
| "odds_ratio_loss": 0.697514533996582, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08293350785970688, | |
| "rewards/margins": 0.012766830623149872, | |
| "rewards/rejected": -0.09570035338401794, | |
| "sft_loss": 0.82933509349823, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 1.6003556345854635, | |
| "grad_norm": 3.0883193016052246, | |
| "learning_rate": 2.2351888258048408e-06, | |
| "logits/chosen": -2.7954063415527344, | |
| "logits/rejected": -2.8361153602600098, | |
| "logps/chosen": -0.8643034100532532, | |
| "logps/rejected": -1.0512335300445557, | |
| "loss": 0.928, | |
| "odds_ratio_loss": 0.6372779011726379, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.08643033355474472, | |
| "rewards/margins": 0.018693022429943085, | |
| "rewards/rejected": -0.1051233559846878, | |
| "sft_loss": 0.8643034100532532, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 1.6181373638586352, | |
| "grad_norm": 2.0690836906433105, | |
| "learning_rate": 2.188913284630584e-06, | |
| "logits/chosen": -2.8729634284973145, | |
| "logits/rejected": -2.8929648399353027, | |
| "logps/chosen": -0.9121950268745422, | |
| "logps/rejected": -1.029121994972229, | |
| "loss": 0.9828, | |
| "odds_ratio_loss": 0.7060950398445129, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.09121949970722198, | |
| "rewards/margins": 0.011692697182297707, | |
| "rewards/rejected": -0.10291220247745514, | |
| "sft_loss": 0.9121950268745422, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 1.635919093131807, | |
| "grad_norm": 3.8324978351593018, | |
| "learning_rate": 2.1427457636675652e-06, | |
| "logits/chosen": -2.862370014190674, | |
| "logits/rejected": -2.88765025138855, | |
| "logps/chosen": -0.8752270936965942, | |
| "logps/rejected": -1.0570242404937744, | |
| "loss": 0.944, | |
| "odds_ratio_loss": 0.6881860494613647, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.0875227078795433, | |
| "rewards/margins": 0.018179720267653465, | |
| "rewards/rejected": -0.10570243746042252, | |
| "sft_loss": 0.8752270936965942, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 1.653700822404979, | |
| "grad_norm": 1.9543755054473877, | |
| "learning_rate": 2.096702293897247e-06, | |
| "logits/chosen": -2.8620665073394775, | |
| "logits/rejected": -2.876176357269287, | |
| "logps/chosen": -0.8246392011642456, | |
| "logps/rejected": -1.1381771564483643, | |
| "loss": 0.8887, | |
| "odds_ratio_loss": 0.6403511166572571, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08246392011642456, | |
| "rewards/margins": 0.03135378286242485, | |
| "rewards/rejected": -0.1138177141547203, | |
| "sft_loss": 0.8246392011642456, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 1.6714825516781509, | |
| "grad_norm": 2.1185944080352783, | |
| "learning_rate": 2.0507988632261672e-06, | |
| "logits/chosen": -2.834115505218506, | |
| "logits/rejected": -2.8819518089294434, | |
| "logps/chosen": -0.8675654530525208, | |
| "logps/rejected": -1.0815333127975464, | |
| "loss": 0.9294, | |
| "odds_ratio_loss": 0.6182416677474976, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.08675654232501984, | |
| "rewards/margins": 0.02139679528772831, | |
| "rewards/rejected": -0.1081533432006836, | |
| "sft_loss": 0.8675654530525208, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 1.6892642809513225, | |
| "grad_norm": 3.2840750217437744, | |
| "learning_rate": 2.005051410934382e-06, | |
| "logits/chosen": -2.8422374725341797, | |
| "logits/rejected": -2.8813586235046387, | |
| "logps/chosen": -0.9204028248786926, | |
| "logps/rejected": -1.0668128728866577, | |
| "loss": 0.9896, | |
| "odds_ratio_loss": 0.69184410572052, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.0920402854681015, | |
| "rewards/margins": 0.01464100182056427, | |
| "rewards/rejected": -0.10668128728866577, | |
| "sft_loss": 0.9204028248786926, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 1.7070460102244942, | |
| "grad_norm": 1.923535704612732, | |
| "learning_rate": 1.9594758221407843e-06, | |
| "logits/chosen": -2.8552603721618652, | |
| "logits/rejected": -2.8695271015167236, | |
| "logps/chosen": -0.8000820279121399, | |
| "logps/rejected": -1.0352107286453247, | |
| "loss": 0.8598, | |
| "odds_ratio_loss": 0.5971243977546692, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.08000819385051727, | |
| "rewards/margins": 0.02351287379860878, | |
| "rewards/rejected": -0.10352107137441635, | |
| "sft_loss": 0.8000820279121399, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 1.724827739497666, | |
| "grad_norm": 3.057365894317627, | |
| "learning_rate": 1.9140879222872408e-06, | |
| "logits/chosen": -2.873203992843628, | |
| "logits/rejected": -2.894540309906006, | |
| "logps/chosen": -0.813400149345398, | |
| "logps/rejected": -0.9625688791275024, | |
| "loss": 0.8825, | |
| "odds_ratio_loss": 0.6909626722335815, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08134002983570099, | |
| "rewards/margins": 0.014916857704520226, | |
| "rewards/rejected": -0.09625686705112457, | |
| "sft_loss": 0.813400149345398, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 1.742609468770838, | |
| "grad_norm": 1.2360864877700806, | |
| "learning_rate": 1.8689034716434346e-06, | |
| "logits/chosen": -2.8773324489593506, | |
| "logits/rejected": -2.8978989124298096, | |
| "logps/chosen": -0.8950971364974976, | |
| "logps/rejected": -1.0365970134735107, | |
| "loss": 0.9651, | |
| "odds_ratio_loss": 0.7001620531082153, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08950972557067871, | |
| "rewards/margins": 0.014149991795420647, | |
| "rewards/rejected": -0.1036597266793251, | |
| "sft_loss": 0.8950971364974976, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 1.76039119804401, | |
| "grad_norm": 1.4812334775924683, | |
| "learning_rate": 1.8239381598343576e-06, | |
| "logits/chosen": -2.8683552742004395, | |
| "logits/rejected": -2.890810012817383, | |
| "logps/chosen": -0.8186439275741577, | |
| "logps/rejected": -1.0320905447006226, | |
| "loss": 0.8854, | |
| "odds_ratio_loss": 0.6674761772155762, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08186439424753189, | |
| "rewards/margins": 0.021344667300581932, | |
| "rewards/rejected": -0.10320906341075897, | |
| "sft_loss": 0.8186439275741577, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 1.7781729273171816, | |
| "grad_norm": 5.4584126472473145, | |
| "learning_rate": 1.779207600392312e-06, | |
| "logits/chosen": -2.8556861877441406, | |
| "logits/rejected": -2.859384298324585, | |
| "logps/chosen": -0.840672492980957, | |
| "logps/rejected": -1.0127156972885132, | |
| "loss": 0.906, | |
| "odds_ratio_loss": 0.6529561877250671, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08406725525856018, | |
| "rewards/margins": 0.01720432937145233, | |
| "rewards/rejected": -0.10127158463001251, | |
| "sft_loss": 0.840672492980957, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.7781729273171816, | |
| "eval_logits/chosen": -2.8532934188842773, | |
| "eval_logits/rejected": -2.8782994747161865, | |
| "eval_logps/chosen": -0.8414809107780457, | |
| "eval_logps/rejected": -1.0811069011688232, | |
| "eval_loss": 0.903913676738739, | |
| "eval_odds_ratio_loss": 0.6243272423744202, | |
| "eval_rewards/accuracies": 0.578000009059906, | |
| "eval_rewards/chosen": -0.0841480940580368, | |
| "eval_rewards/margins": 0.023962605744600296, | |
| "eval_rewards/rejected": -0.1081107035279274, | |
| "eval_runtime": 189.1709, | |
| "eval_samples_per_second": 5.286, | |
| "eval_sft_loss": 0.8414809107780457, | |
| "eval_steps_per_second": 2.643, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.7959546565903532, | |
| "grad_norm": 1.9621540307998657, | |
| "learning_rate": 1.7347273253353552e-06, | |
| "logits/chosen": -2.8437576293945312, | |
| "logits/rejected": -2.8630166053771973, | |
| "logps/chosen": -0.8406645655632019, | |
| "logps/rejected": -1.0365639925003052, | |
| "loss": 0.9076, | |
| "odds_ratio_loss": 0.669218897819519, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08406645804643631, | |
| "rewards/margins": 0.019589943811297417, | |
| "rewards/rejected": -0.10365639626979828, | |
| "sft_loss": 0.8406645655632019, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 1.8137363858635251, | |
| "grad_norm": 4.011701583862305, | |
| "learning_rate": 1.690512779774029e-06, | |
| "logits/chosen": -2.848282814025879, | |
| "logits/rejected": -2.8511271476745605, | |
| "logps/chosen": -0.8997753858566284, | |
| "logps/rejected": -1.1389718055725098, | |
| "loss": 0.9632, | |
| "odds_ratio_loss": 0.6343892812728882, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.08997754007577896, | |
| "rewards/margins": 0.023919641971588135, | |
| "rewards/rejected": -0.11389718949794769, | |
| "sft_loss": 0.8997753858566284, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 1.831518115136697, | |
| "grad_norm": 2.2862370014190674, | |
| "learning_rate": 1.6465793165482838e-06, | |
| "logits/chosen": -2.8141562938690186, | |
| "logits/rejected": -2.8219082355499268, | |
| "logps/chosen": -0.8186345100402832, | |
| "logps/rejected": -1.0088984966278076, | |
| "loss": 0.8807, | |
| "odds_ratio_loss": 0.6206359267234802, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08186344802379608, | |
| "rewards/margins": 0.019026407971978188, | |
| "rewards/rejected": -0.10088986158370972, | |
| "sft_loss": 0.8186345100402832, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 1.849299844409869, | |
| "grad_norm": 2.5129692554473877, | |
| "learning_rate": 1.6029421908964305e-06, | |
| "logits/chosen": -2.821484088897705, | |
| "logits/rejected": -2.8394651412963867, | |
| "logps/chosen": -0.8219555020332336, | |
| "logps/rejected": -1.3423779010772705, | |
| "loss": 0.8817, | |
| "odds_ratio_loss": 0.5971612334251404, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08219555765390396, | |
| "rewards/margins": 0.052042216062545776, | |
| "rewards/rejected": -0.13423778116703033, | |
| "sft_loss": 0.8219555020332336, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 1.8670815736830408, | |
| "grad_norm": 2.674842596054077, | |
| "learning_rate": 1.559616555157985e-06, | |
| "logits/chosen": -2.8907628059387207, | |
| "logits/rejected": -2.8810207843780518, | |
| "logps/chosen": -0.8370941281318665, | |
| "logps/rejected": -1.0264050960540771, | |
| "loss": 0.9034, | |
| "odds_ratio_loss": 0.6629946231842041, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08370941877365112, | |
| "rewards/margins": 0.018931085243821144, | |
| "rewards/rejected": -0.10264050960540771, | |
| "sft_loss": 0.8370941281318665, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 1.8848633029562125, | |
| "grad_norm": 2.72856068611145, | |
| "learning_rate": 1.516617453512252e-06, | |
| "logits/chosen": -2.8834056854248047, | |
| "logits/rejected": -2.8996243476867676, | |
| "logps/chosen": -0.8767944574356079, | |
| "logps/rejected": -1.0179784297943115, | |
| "loss": 0.9475, | |
| "odds_ratio_loss": 0.7070103883743286, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08767944574356079, | |
| "rewards/margins": 0.014118405990302563, | |
| "rewards/rejected": -0.10179785639047623, | |
| "sft_loss": 0.8767944574356079, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 1.9026450322293842, | |
| "grad_norm": 3.5679309368133545, | |
| "learning_rate": 1.473959816754449e-06, | |
| "logits/chosen": -2.857292890548706, | |
| "logits/rejected": -2.8642122745513916, | |
| "logps/chosen": -0.8274821043014526, | |
| "logps/rejected": -0.9466173052787781, | |
| "loss": 0.8975, | |
| "odds_ratio_loss": 0.6997173428535461, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08274821937084198, | |
| "rewards/margins": 0.011913511902093887, | |
| "rewards/rejected": -0.09466172009706497, | |
| "sft_loss": 0.8274821043014526, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 1.920426761502556, | |
| "grad_norm": 1.369982361793518, | |
| "learning_rate": 1.4316584571112213e-06, | |
| "logits/chosen": -2.8926608562469482, | |
| "logits/rejected": -2.9016175270080566, | |
| "logps/chosen": -0.878426194190979, | |
| "logps/rejected": -1.022551417350769, | |
| "loss": 0.9463, | |
| "odds_ratio_loss": 0.6782765984535217, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08784262835979462, | |
| "rewards/margins": 0.014412516728043556, | |
| "rewards/rejected": -0.10225514322519302, | |
| "sft_loss": 0.878426194190979, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 1.938208490775728, | |
| "grad_norm": 2.1955406665802, | |
| "learning_rate": 1.389728063097306e-06, | |
| "logits/chosen": -2.859020471572876, | |
| "logits/rejected": -2.871717929840088, | |
| "logps/chosen": -0.8529576063156128, | |
| "logps/rejected": -1.0697009563446045, | |
| "loss": 0.9165, | |
| "odds_ratio_loss": 0.6351792216300964, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08529576659202576, | |
| "rewards/margins": 0.021674351766705513, | |
| "rewards/rejected": -0.10697011649608612, | |
| "sft_loss": 0.8529576063156128, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 1.9559902200488999, | |
| "grad_norm": 1.758565068244934, | |
| "learning_rate": 1.348183194415179e-06, | |
| "logits/chosen": -2.853273868560791, | |
| "logits/rejected": -2.826066493988037, | |
| "logps/chosen": -0.8589800000190735, | |
| "logps/rejected": -1.1160128116607666, | |
| "loss": 0.9192, | |
| "odds_ratio_loss": 0.6018035411834717, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.0858980119228363, | |
| "rewards/margins": 0.025703275576233864, | |
| "rewards/rejected": -0.11160127818584442, | |
| "sft_loss": 0.8589800000190735, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 1.9737719493220716, | |
| "grad_norm": 2.2166192531585693, | |
| "learning_rate": 1.3070382768994015e-06, | |
| "logits/chosen": -2.8573784828186035, | |
| "logits/rejected": -2.87496018409729, | |
| "logps/chosen": -0.8593405485153198, | |
| "logps/rejected": -1.0153789520263672, | |
| "loss": 0.9242, | |
| "odds_ratio_loss": 0.6489418148994446, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08593406528234482, | |
| "rewards/margins": 0.015603835694491863, | |
| "rewards/rejected": -0.10153790563344955, | |
| "sft_loss": 0.8593405485153198, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 1.9915536785952432, | |
| "grad_norm": 2.127784013748169, | |
| "learning_rate": 1.2663075975074746e-06, | |
| "logits/chosen": -2.8475844860076904, | |
| "logits/rejected": -2.8674192428588867, | |
| "logps/chosen": -0.830818772315979, | |
| "logps/rejected": -1.1525778770446777, | |
| "loss": 0.8944, | |
| "odds_ratio_loss": 0.635840654373169, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.08308187872171402, | |
| "rewards/margins": 0.032175906002521515, | |
| "rewards/rejected": -0.11525777727365494, | |
| "sft_loss": 0.830818772315979, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 2.009335407868415, | |
| "grad_norm": 2.4992928504943848, | |
| "learning_rate": 1.2260052993589034e-06, | |
| "logits/chosen": -2.8519034385681152, | |
| "logits/rejected": -2.8721909523010254, | |
| "logps/chosen": -0.9101880192756653, | |
| "logps/rejected": -1.0249983072280884, | |
| "loss": 0.9847, | |
| "odds_ratio_loss": 0.7451602220535278, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09101879596710205, | |
| "rewards/margins": 0.011481017805635929, | |
| "rewards/rejected": -0.102499820291996, | |
| "sft_loss": 0.9101880192756653, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 2.027117137141587, | |
| "grad_norm": 1.3270055055618286, | |
| "learning_rate": 1.1861453768242099e-06, | |
| "logits/chosen": -2.8831446170806885, | |
| "logits/rejected": -2.8843555450439453, | |
| "logps/chosen": -0.7847625613212585, | |
| "logps/rejected": -1.0357749462127686, | |
| "loss": 0.8449, | |
| "odds_ratio_loss": 0.6014257669448853, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.07847625762224197, | |
| "rewards/margins": 0.025101233273744583, | |
| "rewards/rejected": -0.10357747972011566, | |
| "sft_loss": 0.7847625613212585, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 2.044898866414759, | |
| "grad_norm": 11.792732238769531, | |
| "learning_rate": 1.1467416706655982e-06, | |
| "logits/chosen": -2.8700387477874756, | |
| "logits/rejected": -2.8984532356262207, | |
| "logps/chosen": -0.9160982370376587, | |
| "logps/rejected": -1.110647439956665, | |
| "loss": 0.9851, | |
| "odds_ratio_loss": 0.6902655363082886, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.09160983562469482, | |
| "rewards/margins": 0.019454922527074814, | |
| "rewards/rejected": -0.11106475442647934, | |
| "sft_loss": 0.9160982370376587, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 2.062680595687931, | |
| "grad_norm": 2.206909418106079, | |
| "learning_rate": 1.1078078632309559e-06, | |
| "logits/chosen": -2.8493118286132812, | |
| "logits/rejected": -2.8856310844421387, | |
| "logps/chosen": -0.8342447280883789, | |
| "logps/rejected": -1.0583864450454712, | |
| "loss": 0.8987, | |
| "odds_ratio_loss": 0.644468367099762, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08342447131872177, | |
| "rewards/margins": 0.022414181381464005, | |
| "rewards/rejected": -0.10583865642547607, | |
| "sft_loss": 0.8342447280883789, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 2.0804623249611023, | |
| "grad_norm": 4.428743362426758, | |
| "learning_rate": 1.0693574737028627e-06, | |
| "logits/chosen": -2.860917329788208, | |
| "logits/rejected": -2.8824262619018555, | |
| "logps/chosen": -0.859830379486084, | |
| "logps/rejected": -1.011566162109375, | |
| "loss": 0.9292, | |
| "odds_ratio_loss": 0.6934677362442017, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.08598305284976959, | |
| "rewards/margins": 0.015173576772212982, | |
| "rewards/rejected": -0.10115662962198257, | |
| "sft_loss": 0.859830379486084, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 2.098244054234274, | |
| "grad_norm": 2.3405988216400146, | |
| "learning_rate": 1.0314038534042586e-06, | |
| "logits/chosen": -2.88313364982605, | |
| "logits/rejected": -2.89017915725708, | |
| "logps/chosen": -0.8049169778823853, | |
| "logps/rejected": -1.0722546577453613, | |
| "loss": 0.8704, | |
| "odds_ratio_loss": 0.6548857688903809, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08049169927835464, | |
| "rewards/margins": 0.026733767241239548, | |
| "rewards/rejected": -0.1072254627943039, | |
| "sft_loss": 0.8049169778823853, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 2.116025783507446, | |
| "grad_norm": 3.3044447898864746, | |
| "learning_rate": 9.939601811623946e-07, | |
| "logits/chosen": -2.8709566593170166, | |
| "logits/rejected": -2.8734612464904785, | |
| "logps/chosen": -0.84669029712677, | |
| "logps/rejected": -1.037172555923462, | |
| "loss": 0.9141, | |
| "odds_ratio_loss": 0.6745220422744751, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08466903120279312, | |
| "rewards/margins": 0.019048208370804787, | |
| "rewards/rejected": -0.10371723026037216, | |
| "sft_loss": 0.84669029712677, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 2.133807512780618, | |
| "grad_norm": 1.8569538593292236, | |
| "learning_rate": 9.570394587326825e-07, | |
| "logits/chosen": -2.8537509441375732, | |
| "logits/rejected": -2.8589584827423096, | |
| "logps/chosen": -0.8301420211791992, | |
| "logps/rejected": -1.0919809341430664, | |
| "loss": 0.8916, | |
| "odds_ratio_loss": 0.6140956878662109, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.08301420509815216, | |
| "rewards/margins": 0.02618388459086418, | |
| "rewards/rejected": -0.10919810831546783, | |
| "sft_loss": 0.8301420211791992, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 2.15158924205379, | |
| "grad_norm": 1.6108454465866089, | |
| "learning_rate": 9.206545062840302e-07, | |
| "logits/chosen": -2.9044604301452637, | |
| "logits/rejected": -2.876079559326172, | |
| "logps/chosen": -0.7979404926300049, | |
| "logps/rejected": -1.1050665378570557, | |
| "loss": 0.8573, | |
| "odds_ratio_loss": 0.5938332676887512, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.07979404926300049, | |
| "rewards/margins": 0.03071259893476963, | |
| "rewards/rejected": -0.11050665378570557, | |
| "sft_loss": 0.7979404926300049, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 2.1693709713269618, | |
| "grad_norm": 1.6135528087615967, | |
| "learning_rate": 8.848179579472285e-07, | |
| "logits/chosen": -2.8942551612854004, | |
| "logits/rejected": -2.9058589935302734, | |
| "logps/chosen": -0.8136836886405945, | |
| "logps/rejected": -0.9423009157180786, | |
| "loss": 0.8803, | |
| "odds_ratio_loss": 0.666199803352356, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08136837184429169, | |
| "rewards/margins": 0.012861723080277443, | |
| "rewards/rejected": -0.09423010051250458, | |
| "sft_loss": 0.8136836886405945, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 2.1871527006001332, | |
| "grad_norm": 4.099541664123535, | |
| "learning_rate": 8.495422574279403e-07, | |
| "logits/chosen": -2.8292388916015625, | |
| "logits/rejected": -2.825455904006958, | |
| "logps/chosen": -0.7480133175849915, | |
| "logps/rejected": -1.0116589069366455, | |
| "loss": 0.8073, | |
| "odds_ratio_loss": 0.5924416780471802, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.07480133324861526, | |
| "rewards/margins": 0.026364561170339584, | |
| "rewards/rejected": -0.10116589069366455, | |
| "sft_loss": 0.7480133175849915, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 2.204934429873305, | |
| "grad_norm": 2.086594581604004, | |
| "learning_rate": 8.148396536858063e-07, | |
| "logits/chosen": -2.863175630569458, | |
| "logits/rejected": -2.8580069541931152, | |
| "logps/chosen": -0.8519015312194824, | |
| "logps/rejected": -1.1402422189712524, | |
| "loss": 0.9162, | |
| "odds_ratio_loss": 0.6432971954345703, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08519015461206436, | |
| "rewards/margins": 0.028834056109189987, | |
| "rewards/rejected": -0.11402420699596405, | |
| "sft_loss": 0.8519015312194824, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 2.222716159146477, | |
| "grad_norm": 2.096405506134033, | |
| "learning_rate": 7.807221966811815e-07, | |
| "logits/chosen": -2.8480238914489746, | |
| "logits/rejected": -2.8696630001068115, | |
| "logps/chosen": -0.8801048994064331, | |
| "logps/rejected": -1.0339150428771973, | |
| "loss": 0.9478, | |
| "odds_ratio_loss": 0.6773165464401245, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08801049739122391, | |
| "rewards/margins": 0.015381010249257088, | |
| "rewards/rejected": -0.10339150577783585, | |
| "sft_loss": 0.8801048994064331, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 2.240497888419649, | |
| "grad_norm": 1.47752845287323, | |
| "learning_rate": 7.47201733190962e-07, | |
| "logits/chosen": -2.831873655319214, | |
| "logits/rejected": -2.850604772567749, | |
| "logps/chosen": -0.798612117767334, | |
| "logps/rejected": -1.0020263195037842, | |
| "loss": 0.8614, | |
| "odds_ratio_loss": 0.6277336478233337, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.07986120879650116, | |
| "rewards/margins": 0.020341429859399796, | |
| "rewards/rejected": -0.10020263493061066, | |
| "sft_loss": 0.798612117767334, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 2.258279617692821, | |
| "grad_norm": 1.7826944589614868, | |
| "learning_rate": 7.142899026949721e-07, | |
| "logits/chosen": -2.8982224464416504, | |
| "logits/rejected": -2.9022057056427, | |
| "logps/chosen": -0.817043662071228, | |
| "logps/rejected": -0.9819319844245911, | |
| "loss": 0.8802, | |
| "odds_ratio_loss": 0.6313419938087463, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.08170437067747116, | |
| "rewards/margins": 0.016488831490278244, | |
| "rewards/rejected": -0.0981932133436203, | |
| "sft_loss": 0.817043662071228, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 2.2760613469659923, | |
| "grad_norm": 3.5314130783081055, | |
| "learning_rate": 6.819981333343273e-07, | |
| "logits/chosen": -2.81345796585083, | |
| "logits/rejected": -2.830575704574585, | |
| "logps/chosen": -0.8146475553512573, | |
| "logps/rejected": -1.0825046300888062, | |
| "loss": 0.8761, | |
| "odds_ratio_loss": 0.6145650744438171, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08146476000547409, | |
| "rewards/margins": 0.026785722002387047, | |
| "rewards/rejected": -0.10825047641992569, | |
| "sft_loss": 0.8146475553512573, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 2.293843076239164, | |
| "grad_norm": 2.047950267791748, | |
| "learning_rate": 6.503376379431839e-07, | |
| "logits/chosen": -2.8456294536590576, | |
| "logits/rejected": -2.8468470573425293, | |
| "logps/chosen": -0.8984563946723938, | |
| "logps/rejected": -0.965488612651825, | |
| "loss": 0.9673, | |
| "odds_ratio_loss": 0.68804532289505, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08984563499689102, | |
| "rewards/margins": 0.006703221704810858, | |
| "rewards/rejected": -0.09654886275529861, | |
| "sft_loss": 0.8984563946723938, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 2.311624805512336, | |
| "grad_norm": 1.6956673860549927, | |
| "learning_rate": 6.193194101552502e-07, | |
| "logits/chosen": -2.8512775897979736, | |
| "logits/rejected": -2.8425419330596924, | |
| "logps/chosen": -0.793393611907959, | |
| "logps/rejected": -1.0585198402404785, | |
| "loss": 0.8516, | |
| "odds_ratio_loss": 0.5823057293891907, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.07933936268091202, | |
| "rewards/margins": 0.026512619107961655, | |
| "rewards/rejected": -0.10585198551416397, | |
| "sft_loss": 0.793393611907959, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 2.329406534785508, | |
| "grad_norm": 2.3841018676757812, | |
| "learning_rate": 5.889542205864083e-07, | |
| "logits/chosen": -2.848145008087158, | |
| "logits/rejected": -2.8572099208831787, | |
| "logps/chosen": -0.853177547454834, | |
| "logps/rejected": -1.0807201862335205, | |
| "loss": 0.9165, | |
| "odds_ratio_loss": 0.6334924697875977, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.08531775325536728, | |
| "rewards/margins": 0.022754264995455742, | |
| "rewards/rejected": -0.10807202011346817, | |
| "sft_loss": 0.853177547454834, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 2.34718826405868, | |
| "grad_norm": 1.8613510131835938, | |
| "learning_rate": 5.592526130947862e-07, | |
| "logits/chosen": -2.81400203704834, | |
| "logits/rejected": -2.84773325920105, | |
| "logps/chosen": -0.8714796304702759, | |
| "logps/rejected": -1.0702935457229614, | |
| "loss": 0.9403, | |
| "odds_ratio_loss": 0.6886882781982422, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08714796602725983, | |
| "rewards/margins": 0.019881393760442734, | |
| "rewards/rejected": -0.10702935606241226, | |
| "sft_loss": 0.8714796304702759, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 2.3649699933318518, | |
| "grad_norm": 1.9149771928787231, | |
| "learning_rate": 5.302249011195507e-07, | |
| "logits/chosen": -2.82387375831604, | |
| "logits/rejected": -2.8486294746398926, | |
| "logps/chosen": -0.8160147666931152, | |
| "logps/rejected": -0.9580420255661011, | |
| "loss": 0.8828, | |
| "odds_ratio_loss": 0.6679019927978516, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08160146325826645, | |
| "rewards/margins": 0.014202730730175972, | |
| "rewards/rejected": -0.09580419957637787, | |
| "sft_loss": 0.8160147666931152, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 2.382751722605023, | |
| "grad_norm": 2.322023630142212, | |
| "learning_rate": 5.018811640997307e-07, | |
| "logits/chosen": -2.833300828933716, | |
| "logits/rejected": -2.8586716651916504, | |
| "logps/chosen": -0.8773719072341919, | |
| "logps/rejected": -1.2085764408111572, | |
| "loss": 0.9397, | |
| "odds_ratio_loss": 0.6232184171676636, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.08773718029260635, | |
| "rewards/margins": 0.033120471984148026, | |
| "rewards/rejected": -0.12085764110088348, | |
| "sft_loss": 0.8773719072341919, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 2.400533451878195, | |
| "grad_norm": 1.6597644090652466, | |
| "learning_rate": 4.7423124397427105e-07, | |
| "logits/chosen": -2.8043930530548096, | |
| "logits/rejected": -2.8492674827575684, | |
| "logps/chosen": -0.8693526983261108, | |
| "logps/rejected": -1.019430160522461, | |
| "loss": 0.9355, | |
| "odds_ratio_loss": 0.6619070768356323, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08693526685237885, | |
| "rewards/margins": 0.015007746405899525, | |
| "rewards/rejected": -0.1019430160522461, | |
| "sft_loss": 0.8693526983261108, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 2.418315181151367, | |
| "grad_norm": 2.013772487640381, | |
| "learning_rate": 4.472847417645787e-07, | |
| "logits/chosen": -2.856095552444458, | |
| "logits/rejected": -2.853999614715576, | |
| "logps/chosen": -0.8434507250785828, | |
| "logps/rejected": -1.1554135084152222, | |
| "loss": 0.9027, | |
| "odds_ratio_loss": 0.5929771661758423, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.08434506505727768, | |
| "rewards/margins": 0.031196290627121925, | |
| "rewards/rejected": -0.11554136127233505, | |
| "sft_loss": 0.8434507250785828, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 2.436096910424539, | |
| "grad_norm": 1.1766934394836426, | |
| "learning_rate": 4.210510142406993e-07, | |
| "logits/chosen": -2.826666831970215, | |
| "logits/rejected": -2.8096094131469727, | |
| "logps/chosen": -0.8101630210876465, | |
| "logps/rejected": -1.061128854751587, | |
| "loss": 0.8711, | |
| "odds_ratio_loss": 0.6095449924468994, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.08101630210876465, | |
| "rewards/margins": 0.0250965915620327, | |
| "rewards/rejected": -0.10611288249492645, | |
| "sft_loss": 0.8101630210876465, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 2.4538786396977104, | |
| "grad_norm": 1.6385358572006226, | |
| "learning_rate": 3.9553917067232966e-07, | |
| "logits/chosen": -2.8439018726348877, | |
| "logits/rejected": -2.869356155395508, | |
| "logps/chosen": -0.8570329546928406, | |
| "logps/rejected": -1.0249860286712646, | |
| "loss": 0.9248, | |
| "odds_ratio_loss": 0.6779541373252869, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.0857032984495163, | |
| "rewards/margins": 0.016795307397842407, | |
| "rewards/rejected": -0.1024986058473587, | |
| "sft_loss": 0.8570329546928406, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 2.4716603689708823, | |
| "grad_norm": 1.9620130062103271, | |
| "learning_rate": 3.707580696657509e-07, | |
| "logits/chosen": -2.8321003913879395, | |
| "logits/rejected": -2.8363351821899414, | |
| "logps/chosen": -0.8065320253372192, | |
| "logps/rejected": -0.9883378744125366, | |
| "loss": 0.8702, | |
| "odds_ratio_loss": 0.6370663046836853, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.08065320551395416, | |
| "rewards/margins": 0.018180575221776962, | |
| "rewards/rejected": -0.09883377701044083, | |
| "sft_loss": 0.8065320253372192, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 2.489442098244054, | |
| "grad_norm": 1.9136977195739746, | |
| "learning_rate": 3.4671631608781815e-07, | |
| "logits/chosen": -2.8458282947540283, | |
| "logits/rejected": -2.858309745788574, | |
| "logps/chosen": -0.8568046689033508, | |
| "logps/rejected": -1.1048234701156616, | |
| "loss": 0.9252, | |
| "odds_ratio_loss": 0.6840890645980835, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08568046987056732, | |
| "rewards/margins": 0.02480187825858593, | |
| "rewards/rejected": -0.110482357442379, | |
| "sft_loss": 0.8568046689033508, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 2.507223827517226, | |
| "grad_norm": 2.3483951091766357, | |
| "learning_rate": 3.234222580780405e-07, | |
| "logits/chosen": -2.8452916145324707, | |
| "logits/rejected": -2.8620214462280273, | |
| "logps/chosen": -0.8229707479476929, | |
| "logps/rejected": -1.0171488523483276, | |
| "loss": 0.8863, | |
| "odds_ratio_loss": 0.6332994699478149, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08229707181453705, | |
| "rewards/margins": 0.01941780559718609, | |
| "rewards/rejected": -0.10171488672494888, | |
| "sft_loss": 0.8229707479476929, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 2.525005556790398, | |
| "grad_norm": 1.985844612121582, | |
| "learning_rate": 3.0088398414982375e-07, | |
| "logits/chosen": -2.810109853744507, | |
| "logits/rejected": -2.8480405807495117, | |
| "logps/chosen": -0.8503270149230957, | |
| "logps/rejected": -1.14946711063385, | |
| "loss": 0.9167, | |
| "odds_ratio_loss": 0.6633514165878296, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08503270149230957, | |
| "rewards/margins": 0.02991401217877865, | |
| "rewards/rejected": -0.11494670808315277, | |
| "sft_loss": 0.8503270149230957, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 2.54278728606357, | |
| "grad_norm": 1.8843451738357544, | |
| "learning_rate": 2.7910932038184487e-07, | |
| "logits/chosen": -2.828174352645874, | |
| "logits/rejected": -2.8114073276519775, | |
| "logps/chosen": -0.8486431241035461, | |
| "logps/rejected": -1.0430662631988525, | |
| "loss": 0.9131, | |
| "odds_ratio_loss": 0.6441913843154907, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.08486433327198029, | |
| "rewards/margins": 0.019442306831479073, | |
| "rewards/rejected": -0.10430662333965302, | |
| "sft_loss": 0.8486431241035461, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 2.5605690153367417, | |
| "grad_norm": 1.9914612770080566, | |
| "learning_rate": 2.5810582770057325e-07, | |
| "logits/chosen": -2.873000383377075, | |
| "logits/rejected": -2.902294635772705, | |
| "logps/chosen": -0.8076226115226746, | |
| "logps/rejected": -1.0467280149459839, | |
| "loss": 0.8714, | |
| "odds_ratio_loss": 0.6380000114440918, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.08076226711273193, | |
| "rewards/margins": 0.023910541087388992, | |
| "rewards/rejected": -0.10467280447483063, | |
| "sft_loss": 0.8076226115226746, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 2.578350744609913, | |
| "grad_norm": 2.0563340187072754, | |
| "learning_rate": 2.3788079925484402e-07, | |
| "logits/chosen": -2.849846601486206, | |
| "logits/rejected": -2.8575031757354736, | |
| "logps/chosen": -0.8711313009262085, | |
| "logps/rejected": -0.9900692105293274, | |
| "loss": 0.9391, | |
| "odds_ratio_loss": 0.6800366044044495, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08711313456296921, | |
| "rewards/margins": 0.01189377624541521, | |
| "rewards/rejected": -0.09900690615177155, | |
| "sft_loss": 0.8711313009262085, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 2.596132473883085, | |
| "grad_norm": 2.145470142364502, | |
| "learning_rate": 2.1844125788342661e-07, | |
| "logits/chosen": -2.840655565261841, | |
| "logits/rejected": -2.859443187713623, | |
| "logps/chosen": -0.7942542433738708, | |
| "logps/rejected": -1.2094497680664062, | |
| "loss": 0.8548, | |
| "odds_ratio_loss": 0.6057204008102417, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.07942543178796768, | |
| "rewards/margins": 0.04151954501867294, | |
| "rewards/rejected": -0.12094497680664062, | |
| "sft_loss": 0.7942542433738708, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 2.613914203156257, | |
| "grad_norm": 1.8150006532669067, | |
| "learning_rate": 1.9979395367644428e-07, | |
| "logits/chosen": -2.865077495574951, | |
| "logits/rejected": -2.8804574012756348, | |
| "logps/chosen": -0.8026032447814941, | |
| "logps/rejected": -1.044508457183838, | |
| "loss": 0.8624, | |
| "odds_ratio_loss": 0.5979448556900024, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08026032149791718, | |
| "rewards/margins": 0.024190524592995644, | |
| "rewards/rejected": -0.10445085912942886, | |
| "sft_loss": 0.8026032447814941, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 2.631695932429429, | |
| "grad_norm": 3.1679821014404297, | |
| "learning_rate": 1.81945361631512e-07, | |
| "logits/chosen": -2.8732008934020996, | |
| "logits/rejected": -2.8841676712036133, | |
| "logps/chosen": -0.8442559242248535, | |
| "logps/rejected": -1.0102087259292603, | |
| "loss": 0.9134, | |
| "odds_ratio_loss": 0.6910725235939026, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.08442559093236923, | |
| "rewards/margins": 0.016595274209976196, | |
| "rewards/rejected": -0.10102085769176483, | |
| "sft_loss": 0.8442559242248535, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 2.6494776617026004, | |
| "grad_norm": 2.5551412105560303, | |
| "learning_rate": 1.6490167940538343e-07, | |
| "logits/chosen": -2.8778164386749268, | |
| "logits/rejected": -2.8786826133728027, | |
| "logps/chosen": -0.8008125424385071, | |
| "logps/rejected": -1.0858347415924072, | |
| "loss": 0.8593, | |
| "odds_ratio_loss": 0.5846473574638367, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.08008125424385071, | |
| "rewards/margins": 0.028502214699983597, | |
| "rewards/rejected": -0.1085834726691246, | |
| "sft_loss": 0.8008125424385071, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 2.6672593909757722, | |
| "grad_norm": 1.7907795906066895, | |
| "learning_rate": 1.4866882516191339e-07, | |
| "logits/chosen": -2.795592784881592, | |
| "logits/rejected": -2.8337063789367676, | |
| "logps/chosen": -0.8366915583610535, | |
| "logps/rejected": -1.0835788249969482, | |
| "loss": 0.9019, | |
| "odds_ratio_loss": 0.652554988861084, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08366917073726654, | |
| "rewards/margins": 0.0246887169778347, | |
| "rewards/rejected": -0.10835788398981094, | |
| "sft_loss": 0.8366915583610535, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.6672593909757722, | |
| "eval_logits/chosen": -2.8460819721221924, | |
| "eval_logits/rejected": -2.8720545768737793, | |
| "eval_logps/chosen": -0.8352006673812866, | |
| "eval_logps/rejected": -1.0736578702926636, | |
| "eval_loss": 0.8975116014480591, | |
| "eval_odds_ratio_loss": 0.6231085658073425, | |
| "eval_rewards/accuracies": 0.5899999737739563, | |
| "eval_rewards/chosen": -0.0835200697183609, | |
| "eval_rewards/margins": 0.023845719173550606, | |
| "eval_rewards/rejected": -0.10736579447984695, | |
| "eval_runtime": 189.1942, | |
| "eval_samples_per_second": 5.286, | |
| "eval_sft_loss": 0.8352006673812866, | |
| "eval_steps_per_second": 2.643, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.685041120248944, | |
| "grad_norm": 5.948815822601318, | |
| "learning_rate": 1.3325243551706057e-07, | |
| "logits/chosen": -2.823535442352295, | |
| "logits/rejected": -2.8570055961608887, | |
| "logps/chosen": -0.8226287961006165, | |
| "logps/rejected": -1.2481344938278198, | |
| "loss": 0.8823, | |
| "odds_ratio_loss": 0.5963335633277893, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08226287364959717, | |
| "rewards/margins": 0.04255058616399765, | |
| "rewards/rejected": -0.12481345981359482, | |
| "sft_loss": 0.8226287961006165, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 2.702822849522116, | |
| "grad_norm": 2.1956028938293457, | |
| "learning_rate": 1.1865786358165737e-07, | |
| "logits/chosen": -2.841465473175049, | |
| "logits/rejected": -2.881883382797241, | |
| "logps/chosen": -0.8048586845397949, | |
| "logps/rejected": -0.9951068162918091, | |
| "loss": 0.8691, | |
| "odds_ratio_loss": 0.6420662999153137, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08048586547374725, | |
| "rewards/margins": 0.019024807959794998, | |
| "rewards/rejected": -0.09951067715883255, | |
| "sft_loss": 0.8048586845397949, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 2.720604578795288, | |
| "grad_norm": 4.919430255889893, | |
| "learning_rate": 1.0489017710262311e-07, | |
| "logits/chosen": -2.8700149059295654, | |
| "logits/rejected": -2.9017858505249023, | |
| "logps/chosen": -0.8849034309387207, | |
| "logps/rejected": -1.1535307168960571, | |
| "loss": 0.9517, | |
| "odds_ratio_loss": 0.667965292930603, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.08849034458398819, | |
| "rewards/margins": 0.026862725615501404, | |
| "rewards/rejected": -0.11535308510065079, | |
| "sft_loss": 0.8849034309387207, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 2.73838630806846, | |
| "grad_norm": 2.982793092727661, | |
| "learning_rate": 9.195415670326446e-08, | |
| "logits/chosen": -2.874678611755371, | |
| "logits/rejected": -2.887955904006958, | |
| "logps/chosen": -0.828321099281311, | |
| "logps/rejected": -1.0579955577850342, | |
| "loss": 0.8916, | |
| "odds_ratio_loss": 0.6329658031463623, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08283210545778275, | |
| "rewards/margins": 0.02296745963394642, | |
| "rewards/rejected": -0.10579957067966461, | |
| "sft_loss": 0.828321099281311, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 2.7561680373416317, | |
| "grad_norm": 2.7303407192230225, | |
| "learning_rate": 7.985429422327384e-08, | |
| "logits/chosen": -2.8821816444396973, | |
| "logits/rejected": -2.9013893604278564, | |
| "logps/chosen": -0.8389023542404175, | |
| "logps/rejected": -0.9915286302566528, | |
| "loss": 0.9067, | |
| "odds_ratio_loss": 0.6778423190116882, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08389024436473846, | |
| "rewards/margins": 0.015262606553733349, | |
| "rewards/rejected": -0.09915284812450409, | |
| "sft_loss": 0.8389023542404175, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 2.773949766614803, | |
| "grad_norm": 1.570169448852539, | |
| "learning_rate": 6.859479115900818e-08, | |
| "logits/chosen": -2.848161458969116, | |
| "logits/rejected": -2.8777124881744385, | |
| "logps/chosen": -0.8677851557731628, | |
| "logps/rejected": -1.1111342906951904, | |
| "loss": 0.9293, | |
| "odds_ratio_loss": 0.6150751709938049, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.08677852153778076, | |
| "rewards/margins": 0.024334916844964027, | |
| "rewards/rejected": -0.11111342906951904, | |
| "sft_loss": 0.8677851557731628, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 2.791731495887975, | |
| "grad_norm": 6.0747294425964355, | |
| "learning_rate": 5.817955720457902e-08, | |
| "logits/chosen": -2.8411552906036377, | |
| "logits/rejected": -2.8624863624572754, | |
| "logps/chosen": -0.881393313407898, | |
| "logps/rejected": -1.1098216772079468, | |
| "loss": 0.9455, | |
| "odds_ratio_loss": 0.641205370426178, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.08813934028148651, | |
| "rewards/margins": 0.02284284308552742, | |
| "rewards/rejected": -0.11098217964172363, | |
| "sft_loss": 0.881393313407898, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 2.809513225161147, | |
| "grad_norm": 3.2441201210021973, | |
| "learning_rate": 4.861220889427199e-08, | |
| "logits/chosen": -2.8530640602111816, | |
| "logits/rejected": -2.865347385406494, | |
| "logps/chosen": -0.8542373776435852, | |
| "logps/rejected": -1.0531483888626099, | |
| "loss": 0.9195, | |
| "odds_ratio_loss": 0.6527055501937866, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08542371541261673, | |
| "rewards/margins": 0.019891122356057167, | |
| "rewards/rejected": -0.10531485080718994, | |
| "sft_loss": 0.8542373776435852, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 2.827294954434319, | |
| "grad_norm": 2.0825612545013428, | |
| "learning_rate": 3.9896068346758074e-08, | |
| "logits/chosen": -2.8629543781280518, | |
| "logits/rejected": -2.887873649597168, | |
| "logps/chosen": -0.8457245826721191, | |
| "logps/rejected": -1.0121568441390991, | |
| "loss": 0.9115, | |
| "odds_ratio_loss": 0.6578742265701294, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.0845724567770958, | |
| "rewards/margins": 0.016643229871988297, | |
| "rewards/rejected": -0.10121568292379379, | |
| "sft_loss": 0.8457245826721191, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 2.8450766837074903, | |
| "grad_norm": 5.224942207336426, | |
| "learning_rate": 3.203416211153832e-08, | |
| "logits/chosen": -2.8476452827453613, | |
| "logits/rejected": -2.893502712249756, | |
| "logps/chosen": -0.843380331993103, | |
| "logps/rejected": -1.0194734334945679, | |
| "loss": 0.9131, | |
| "odds_ratio_loss": 0.6969292163848877, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.08433803915977478, | |
| "rewards/margins": 0.017609301954507828, | |
| "rewards/rejected": -0.1019473448395729, | |
| "sft_loss": 0.843380331993103, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 2.8628584129806622, | |
| "grad_norm": 2.344214677810669, | |
| "learning_rate": 2.5029220118019393e-08, | |
| "logits/chosen": -2.8379952907562256, | |
| "logits/rejected": -2.8665690422058105, | |
| "logps/chosen": -0.8621314167976379, | |
| "logps/rejected": -0.9904851913452148, | |
| "loss": 0.929, | |
| "odds_ratio_loss": 0.6684004664421082, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.08621314913034439, | |
| "rewards/margins": 0.01283537782728672, | |
| "rewards/rejected": -0.09904851764440536, | |
| "sft_loss": 0.8621314167976379, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 2.880640142253834, | |
| "grad_norm": 3.038189649581909, | |
| "learning_rate": 1.8883674727586122e-08, | |
| "logits/chosen": -2.868648052215576, | |
| "logits/rejected": -2.876978635787964, | |
| "logps/chosen": -0.8205513954162598, | |
| "logps/rejected": -1.1327790021896362, | |
| "loss": 0.8816, | |
| "odds_ratio_loss": 0.6100881099700928, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.08205513656139374, | |
| "rewards/margins": 0.031222760677337646, | |
| "rewards/rejected": -0.11327788978815079, | |
| "sft_loss": 0.8205513954162598, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 2.898421871527006, | |
| "grad_norm": 1.2839456796646118, | |
| "learning_rate": 1.3599659889000639e-08, | |
| "logits/chosen": -2.8628990650177, | |
| "logits/rejected": -2.881967067718506, | |
| "logps/chosen": -0.7933380007743835, | |
| "logps/rejected": -0.9434422254562378, | |
| "loss": 0.8593, | |
| "odds_ratio_loss": 0.6599572896957397, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.07933380454778671, | |
| "rewards/margins": 0.015010423958301544, | |
| "rewards/rejected": -0.09434423595666885, | |
| "sft_loss": 0.7933380007743835, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 2.916203600800178, | |
| "grad_norm": 1.2355436086654663, | |
| "learning_rate": 9.179010397421528e-09, | |
| "logits/chosen": -2.864149808883667, | |
| "logits/rejected": -2.8793797492980957, | |
| "logps/chosen": -0.8156276941299438, | |
| "logps/rejected": -1.0415749549865723, | |
| "loss": 0.8816, | |
| "odds_ratio_loss": 0.659904956817627, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.08156277239322662, | |
| "rewards/margins": 0.022594723850488663, | |
| "rewards/rejected": -0.10415749251842499, | |
| "sft_loss": 0.8156276941299438, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 2.93398533007335, | |
| "grad_norm": 1.7059450149536133, | |
| "learning_rate": 5.623261257296509e-09, | |
| "logits/chosen": -2.8292007446289062, | |
| "logits/rejected": -2.865652561187744, | |
| "logps/chosen": -0.7724882364273071, | |
| "logps/rejected": -0.9592376947402954, | |
| "loss": 0.8358, | |
| "odds_ratio_loss": 0.6333457231521606, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.07724882662296295, | |
| "rewards/margins": 0.01867493987083435, | |
| "rewards/rejected": -0.0959237664937973, | |
| "sft_loss": 0.7724882364273071, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 2.9517670593465217, | |
| "grad_norm": 2.2234692573547363, | |
| "learning_rate": 2.933647149357122e-09, | |
| "logits/chosen": -2.8548574447631836, | |
| "logits/rejected": -2.8638224601745605, | |
| "logps/chosen": -0.8041044473648071, | |
| "logps/rejected": -1.1135035753250122, | |
| "loss": 0.8679, | |
| "odds_ratio_loss": 0.6383166909217834, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.0804104432463646, | |
| "rewards/margins": 0.030939901247620583, | |
| "rewards/rejected": -0.11135034263134003, | |
| "sft_loss": 0.8041044473648071, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 2.969548788619693, | |
| "grad_norm": 2.0396206378936768, | |
| "learning_rate": 1.1111020018930717e-09, | |
| "logits/chosen": -2.863762617111206, | |
| "logits/rejected": -2.8730015754699707, | |
| "logps/chosen": -0.8626953363418579, | |
| "logps/rejected": -0.9779960513114929, | |
| "loss": 0.9301, | |
| "odds_ratio_loss": 0.6735736131668091, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08626954257488251, | |
| "rewards/margins": 0.011530078016221523, | |
| "rewards/rejected": -0.09779961407184601, | |
| "sft_loss": 0.8626953363418579, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 2.987330517892865, | |
| "grad_norm": 1.7169232368469238, | |
| "learning_rate": 1.5625866646051813e-10, | |
| "logits/chosen": -2.8806686401367188, | |
| "logits/rejected": -2.874525547027588, | |
| "logps/chosen": -0.8173590898513794, | |
| "logps/rejected": -1.051163911819458, | |
| "loss": 0.8775, | |
| "odds_ratio_loss": 0.600918173789978, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.08173591643571854, | |
| "rewards/margins": 0.0233804639428854, | |
| "rewards/rejected": -0.10511638224124908, | |
| "sft_loss": 0.8173590898513794, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 2.997999555456768, | |
| "step": 1686, | |
| "total_flos": 1.985199772705751e+18, | |
| "train_loss": 0.9645641791862949, | |
| "train_runtime": 17170.3123, | |
| "train_samples_per_second": 1.572, | |
| "train_steps_per_second": 0.098 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1686, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "total_flos": 1.985199772705751e+18, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |