| { | |
| "best_metric": 1.0073015689849854, | |
| "best_model_checkpoint": "saves/Vicuna-7B-v1.5/lora/orpo/checkpoint-1500", | |
| "epoch": 2.997999555456768, | |
| "eval_steps": 500, | |
| "global_step": 1686, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.017781729273171815, | |
| "grad_norm": 0.3158996105194092, | |
| "learning_rate": 4.9995745934141085e-06, | |
| "logits/chosen": -0.7898403406143188, | |
| "logits/rejected": -0.7731221914291382, | |
| "logps/chosen": -1.1474043130874634, | |
| "logps/rejected": -1.2031431198120117, | |
| "loss": 1.227, | |
| "odds_ratio_loss": 0.7959282994270325, | |
| "rewards/accuracies": 0.4124999940395355, | |
| "rewards/chosen": -0.11474044620990753, | |
| "rewards/margins": 0.005573858506977558, | |
| "rewards/rejected": -0.12031430006027222, | |
| "sft_loss": 1.1474043130874634, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.03556345854634363, | |
| "grad_norm": 0.8646821975708008, | |
| "learning_rate": 4.9982812903243405e-06, | |
| "logits/chosen": -0.7618139982223511, | |
| "logits/rejected": -0.7260042428970337, | |
| "logps/chosen": -0.9931285977363586, | |
| "logps/rejected": -1.050875186920166, | |
| "loss": 1.0707, | |
| "odds_ratio_loss": 0.7757659554481506, | |
| "rewards/accuracies": 0.45625001192092896, | |
| "rewards/chosen": -0.09931285679340363, | |
| "rewards/margins": 0.005774644669145346, | |
| "rewards/rejected": -0.10508750379085541, | |
| "sft_loss": 0.9931285977363586, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.05334518781951545, | |
| "grad_norm": 0.2927573025226593, | |
| "learning_rate": 4.996120496405222e-06, | |
| "logits/chosen": -0.7767494916915894, | |
| "logits/rejected": -0.7559677362442017, | |
| "logps/chosen": -1.040177345275879, | |
| "logps/rejected": -1.2401186227798462, | |
| "loss": 1.1087, | |
| "odds_ratio_loss": 0.6853717565536499, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.10401773452758789, | |
| "rewards/margins": 0.019994117319583893, | |
| "rewards/rejected": -0.12401185184717178, | |
| "sft_loss": 1.040177345275879, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.07112691709268726, | |
| "grad_norm": 0.3339848518371582, | |
| "learning_rate": 4.99309296196014e-06, | |
| "logits/chosen": -0.7875353693962097, | |
| "logits/rejected": -0.7857375741004944, | |
| "logps/chosen": -1.0764983892440796, | |
| "logps/rejected": -1.1753004789352417, | |
| "loss": 1.1498, | |
| "odds_ratio_loss": 0.7328984141349792, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.10764984041452408, | |
| "rewards/margins": 0.009880214929580688, | |
| "rewards/rejected": -0.11753007024526596, | |
| "sft_loss": 1.0764983892440796, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.08890864636585907, | |
| "grad_norm": 0.3153611719608307, | |
| "learning_rate": 4.989199738255166e-06, | |
| "logits/chosen": -0.7786640524864197, | |
| "logits/rejected": -0.7964621782302856, | |
| "logps/chosen": -1.0476799011230469, | |
| "logps/rejected": -1.1452114582061768, | |
| "loss": 1.1221, | |
| "odds_ratio_loss": 0.7446193099021912, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.10476799309253693, | |
| "rewards/margins": 0.009753172285854816, | |
| "rewards/rejected": -0.11452116817235947, | |
| "sft_loss": 1.0476799011230469, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.1066903756390309, | |
| "grad_norm": 2.7500874996185303, | |
| "learning_rate": 4.984442177154031e-06, | |
| "logits/chosen": -0.7653383612632751, | |
| "logits/rejected": -0.7529075741767883, | |
| "logps/chosen": -1.1525957584381104, | |
| "logps/rejected": -1.2310835123062134, | |
| "loss": 1.2305, | |
| "odds_ratio_loss": 0.7788733243942261, | |
| "rewards/accuracies": 0.45625001192092896, | |
| "rewards/chosen": -0.1152595728635788, | |
| "rewards/margins": 0.007848784327507019, | |
| "rewards/rejected": -0.12310836464166641, | |
| "sft_loss": 1.1525957584381104, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.12447210491220272, | |
| "grad_norm": 0.3525276184082031, | |
| "learning_rate": 4.978821930648704e-06, | |
| "logits/chosen": -0.8071187734603882, | |
| "logits/rejected": -0.7696810364723206, | |
| "logps/chosen": -1.0399789810180664, | |
| "logps/rejected": -1.0721027851104736, | |
| "loss": 1.1208, | |
| "odds_ratio_loss": 0.8085241317749023, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.10399790853261948, | |
| "rewards/margins": 0.003212365787476301, | |
| "rewards/rejected": -0.10721027851104736, | |
| "sft_loss": 1.0399789810180664, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.14225383418537452, | |
| "grad_norm": 0.6355476379394531, | |
| "learning_rate": 4.97234095028576e-06, | |
| "logits/chosen": -0.738179624080658, | |
| "logits/rejected": -0.7453175783157349, | |
| "logps/chosen": -1.1585901975631714, | |
| "logps/rejected": -1.2273097038269043, | |
| "loss": 1.2343, | |
| "odds_ratio_loss": 0.7569113969802856, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.1158590167760849, | |
| "rewards/margins": 0.0068719410337507725, | |
| "rewards/rejected": -0.12273095548152924, | |
| "sft_loss": 1.1585901975631714, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.16003556345854633, | |
| "grad_norm": 0.2942532002925873, | |
| "learning_rate": 4.965001486488743e-06, | |
| "logits/chosen": -0.7591525316238403, | |
| "logits/rejected": -0.7494860887527466, | |
| "logps/chosen": -1.0791616439819336, | |
| "logps/rejected": -1.2336231470108032, | |
| "loss": 1.1471, | |
| "odds_ratio_loss": 0.6791869401931763, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.10791617631912231, | |
| "rewards/margins": 0.015446141362190247, | |
| "rewards/rejected": -0.12336231768131256, | |
| "sft_loss": 1.0791616439819336, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.17781729273171815, | |
| "grad_norm": 0.35266247391700745, | |
| "learning_rate": 4.956806087776732e-06, | |
| "logits/chosen": -0.6999791860580444, | |
| "logits/rejected": -0.6948890686035156, | |
| "logps/chosen": -1.0402957201004028, | |
| "logps/rejected": -1.2390520572662354, | |
| "loss": 1.1124, | |
| "odds_ratio_loss": 0.7215061187744141, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.10402955859899521, | |
| "rewards/margins": 0.01987563632428646, | |
| "rewards/rejected": -0.12390519678592682, | |
| "sft_loss": 1.0402957201004028, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.19559902200489, | |
| "grad_norm": 0.4545610845088959, | |
| "learning_rate": 4.947757599879411e-06, | |
| "logits/chosen": -0.7189663052558899, | |
| "logits/rejected": -0.6851673126220703, | |
| "logps/chosen": -1.147323489189148, | |
| "logps/rejected": -1.289452314376831, | |
| "loss": 1.2227, | |
| "odds_ratio_loss": 0.7533982396125793, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.11473236232995987, | |
| "rewards/margins": 0.014212870970368385, | |
| "rewards/rejected": -0.1289452314376831, | |
| "sft_loss": 1.147323489189148, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.2133807512780618, | |
| "grad_norm": 0.6324980854988098, | |
| "learning_rate": 4.937859164748931e-06, | |
| "logits/chosen": -0.7043695449829102, | |
| "logits/rejected": -0.6795639991760254, | |
| "logps/chosen": -1.0146863460540771, | |
| "logps/rejected": -1.0826324224472046, | |
| "loss": 1.0907, | |
| "odds_ratio_loss": 0.760542094707489, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.10146863758563995, | |
| "rewards/margins": 0.006794607732445002, | |
| "rewards/rejected": -0.10826325416564941, | |
| "sft_loss": 1.0146863460540771, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.23116248055123362, | |
| "grad_norm": 0.4255826771259308, | |
| "learning_rate": 4.92711421946891e-06, | |
| "logits/chosen": -0.6701909899711609, | |
| "logits/rejected": -0.7547520995140076, | |
| "logps/chosen": -1.0397005081176758, | |
| "logps/rejected": -1.1938796043395996, | |
| "loss": 1.1117, | |
| "odds_ratio_loss": 0.7198113799095154, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.10397003591060638, | |
| "rewards/margins": 0.015417915768921375, | |
| "rewards/rejected": -0.11938796192407608, | |
| "sft_loss": 1.0397005081176758, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.24894420982440543, | |
| "grad_norm": 0.7161264419555664, | |
| "learning_rate": 4.915526495060961e-06, | |
| "logits/chosen": -0.6202753782272339, | |
| "logits/rejected": -0.64984530210495, | |
| "logps/chosen": -1.0066936016082764, | |
| "logps/rejected": -1.1723135709762573, | |
| "loss": 1.0745, | |
| "odds_ratio_loss": 0.6777721643447876, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.10066936165094376, | |
| "rewards/margins": 0.016561999917030334, | |
| "rewards/rejected": -0.1172313541173935, | |
| "sft_loss": 1.0066936016082764, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.26672593909757725, | |
| "grad_norm": 0.540038526058197, | |
| "learning_rate": 4.903100015189153e-06, | |
| "logits/chosen": -0.5942473411560059, | |
| "logits/rejected": -0.5408576726913452, | |
| "logps/chosen": -0.9665758013725281, | |
| "logps/rejected": -1.1337311267852783, | |
| "loss": 1.0386, | |
| "odds_ratio_loss": 0.719926118850708, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.09665757417678833, | |
| "rewards/margins": 0.01671554148197174, | |
| "rewards/rejected": -0.11337311565876007, | |
| "sft_loss": 0.9665758013725281, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.28450766837074903, | |
| "grad_norm": 2.370271682739258, | |
| "learning_rate": 4.889839094762848e-06, | |
| "logits/chosen": -0.5599099397659302, | |
| "logits/rejected": -0.5666571855545044, | |
| "logps/chosen": -1.0475890636444092, | |
| "logps/rejected": -1.1946136951446533, | |
| "loss": 1.1206, | |
| "odds_ratio_loss": 0.7300440073013306, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.10475890338420868, | |
| "rewards/margins": 0.014702451415359974, | |
| "rewards/rejected": -0.11946137249469757, | |
| "sft_loss": 1.0475890636444092, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.3022893976439209, | |
| "grad_norm": 0.37259843945503235, | |
| "learning_rate": 4.875748338438416e-06, | |
| "logits/chosen": -0.5827142000198364, | |
| "logits/rejected": -0.5626250505447388, | |
| "logps/chosen": -0.9911508560180664, | |
| "logps/rejected": -1.0813571214675903, | |
| "loss": 1.0632, | |
| "odds_ratio_loss": 0.720399022102356, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.09911508113145828, | |
| "rewards/margins": 0.009020629338920116, | |
| "rewards/rejected": -0.10813571512699127, | |
| "sft_loss": 0.9911508560180664, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.32007112691709266, | |
| "grad_norm": 0.3821701109409332, | |
| "learning_rate": 4.8608326390203386e-06, | |
| "logits/chosen": -0.6059321165084839, | |
| "logits/rejected": -0.5918234586715698, | |
| "logps/chosen": -0.9553475379943848, | |
| "logps/rejected": -1.1111819744110107, | |
| "loss": 1.0245, | |
| "odds_ratio_loss": 0.6911659240722656, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.09553476423025131, | |
| "rewards/margins": 0.01558343879878521, | |
| "rewards/rejected": -0.11111819744110107, | |
| "sft_loss": 0.9553475379943848, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.3378528561902645, | |
| "grad_norm": 0.3977317810058594, | |
| "learning_rate": 4.845097175762251e-06, | |
| "logits/chosen": -0.49882182478904724, | |
| "logits/rejected": -0.48370814323425293, | |
| "logps/chosen": -0.989281952381134, | |
| "logps/rejected": -1.0615712404251099, | |
| "loss": 1.0617, | |
| "odds_ratio_loss": 0.7244290113449097, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.09892819821834564, | |
| "rewards/margins": 0.007228921167552471, | |
| "rewards/rejected": -0.10615710914134979, | |
| "sft_loss": 0.989281952381134, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.3556345854634363, | |
| "grad_norm": 0.46290695667266846, | |
| "learning_rate": 4.8285474125685286e-06, | |
| "logits/chosen": -0.518696129322052, | |
| "logits/rejected": -0.5193291306495667, | |
| "logps/chosen": -1.1205590963363647, | |
| "logps/rejected": -1.1714627742767334, | |
| "loss": 1.198, | |
| "odds_ratio_loss": 0.7740126252174377, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.11205589771270752, | |
| "rewards/margins": 0.00509037496522069, | |
| "rewards/rejected": -0.11714627593755722, | |
| "sft_loss": 1.1205590963363647, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.37341631473660813, | |
| "grad_norm": 0.32425227761268616, | |
| "learning_rate": 4.811189096097025e-06, | |
| "logits/chosen": -0.5530649423599243, | |
| "logits/rejected": -0.5483794808387756, | |
| "logps/chosen": -0.9994535446166992, | |
| "logps/rejected": -1.1620233058929443, | |
| "loss": 1.0712, | |
| "odds_ratio_loss": 0.7175347208976746, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09994535893201828, | |
| "rewards/margins": 0.01625697687268257, | |
| "rewards/rejected": -0.11620233952999115, | |
| "sft_loss": 0.9994535446166992, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.39119804400978, | |
| "grad_norm": 0.5374495387077332, | |
| "learning_rate": 4.793028253763633e-06, | |
| "logits/chosen": -0.46489372849464417, | |
| "logits/rejected": -0.49711689352989197, | |
| "logps/chosen": -0.9644722938537598, | |
| "logps/rejected": -1.098311185836792, | |
| "loss": 1.0422, | |
| "odds_ratio_loss": 0.7768682837486267, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09644722938537598, | |
| "rewards/margins": 0.013383878394961357, | |
| "rewards/rejected": -0.10983110964298248, | |
| "sft_loss": 0.9644722938537598, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.40897977328295176, | |
| "grad_norm": 0.7932880520820618, | |
| "learning_rate": 4.774071191649352e-06, | |
| "logits/chosen": -0.5470231771469116, | |
| "logits/rejected": -0.5435986518859863, | |
| "logps/chosen": -0.9579310417175293, | |
| "logps/rejected": -1.1810802221298218, | |
| "loss": 1.0212, | |
| "odds_ratio_loss": 0.6330138444900513, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.09579310566186905, | |
| "rewards/margins": 0.02231491729617119, | |
| "rewards/rejected": -0.11810803413391113, | |
| "sft_loss": 0.9579310417175293, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.4267615025561236, | |
| "grad_norm": 0.618280291557312, | |
| "learning_rate": 4.7543244923105975e-06, | |
| "logits/chosen": -0.5025745630264282, | |
| "logits/rejected": -0.4722610414028168, | |
| "logps/chosen": -1.0212466716766357, | |
| "logps/rejected": -1.0026448965072632, | |
| "loss": 1.1058, | |
| "odds_ratio_loss": 0.8450964093208313, | |
| "rewards/accuracies": 0.45625001192092896, | |
| "rewards/chosen": -0.1021246686577797, | |
| "rewards/margins": -0.0018601752817630768, | |
| "rewards/rejected": -0.10026448965072632, | |
| "sft_loss": 1.0212466716766357, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.4445432318292954, | |
| "grad_norm": 0.39385247230529785, | |
| "learning_rate": 4.733795012493506e-06, | |
| "logits/chosen": -0.5138652324676514, | |
| "logits/rejected": -0.4715350270271301, | |
| "logps/chosen": -1.0123497247695923, | |
| "logps/rejected": -1.13383150100708, | |
| "loss": 1.0857, | |
| "odds_ratio_loss": 0.7335414886474609, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.10123495757579803, | |
| "rewards/margins": 0.012148191221058369, | |
| "rewards/rejected": -0.11338315904140472, | |
| "sft_loss": 1.0123497247695923, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.46232496110246724, | |
| "grad_norm": 0.3666248619556427, | |
| "learning_rate": 4.712489880753035e-06, | |
| "logits/chosen": -0.3967147171497345, | |
| "logits/rejected": -0.3805852234363556, | |
| "logps/chosen": -0.946629524230957, | |
| "logps/rejected": -1.0246347188949585, | |
| "loss": 1.0164, | |
| "odds_ratio_loss": 0.6973500847816467, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.09466294944286346, | |
| "rewards/margins": 0.007800529710948467, | |
| "rewards/rejected": -0.1024634838104248, | |
| "sft_loss": 0.946629524230957, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.480106690375639, | |
| "grad_norm": 0.6196191906929016, | |
| "learning_rate": 4.690416494977673e-06, | |
| "logits/chosen": -0.3590370714664459, | |
| "logits/rejected": -0.3209628164768219, | |
| "logps/chosen": -0.9477987289428711, | |
| "logps/rejected": -1.1744658946990967, | |
| "loss": 1.0133, | |
| "odds_ratio_loss": 0.654593825340271, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.09477987140417099, | |
| "rewards/margins": 0.02266671508550644, | |
| "rewards/rejected": -0.11744660139083862, | |
| "sft_loss": 0.9477987289428711, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.49788841964881086, | |
| "grad_norm": 0.38255006074905396, | |
| "learning_rate": 4.667582519820639e-06, | |
| "logits/chosen": -0.4478569030761719, | |
| "logits/rejected": -0.40335726737976074, | |
| "logps/chosen": -1.0600357055664062, | |
| "logps/rejected": -1.0844862461090088, | |
| "loss": 1.1374, | |
| "odds_ratio_loss": 0.7734627723693848, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.10600356757640839, | |
| "rewards/margins": 0.002445052145048976, | |
| "rewards/rejected": -0.10844862461090088, | |
| "sft_loss": 1.0600357055664062, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.5156701489219827, | |
| "grad_norm": 0.6143254637718201, | |
| "learning_rate": 4.643995884038443e-06, | |
| "logits/chosen": -0.42634057998657227, | |
| "logits/rejected": -0.4024909436702728, | |
| "logps/chosen": -1.0625637769699097, | |
| "logps/rejected": -1.2203805446624756, | |
| "loss": 1.1314, | |
| "odds_ratio_loss": 0.6885315179824829, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.1062563806772232, | |
| "rewards/margins": 0.01578168198466301, | |
| "rewards/rejected": -0.12203805148601532, | |
| "sft_loss": 1.0625637769699097, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.5334518781951545, | |
| "grad_norm": 0.3366183042526245, | |
| "learning_rate": 4.6196647777377475e-06, | |
| "logits/chosen": -0.37543022632598877, | |
| "logits/rejected": -0.3797139525413513, | |
| "logps/chosen": -0.9299192428588867, | |
| "logps/rejected": -0.9767643213272095, | |
| "loss": 1.0053, | |
| "odds_ratio_loss": 0.7540563344955444, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.09299192577600479, | |
| "rewards/margins": 0.004684499930590391, | |
| "rewards/rejected": -0.09767641872167587, | |
| "sft_loss": 0.9299192428588867, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.5512336074683263, | |
| "grad_norm": 0.5256261825561523, | |
| "learning_rate": 4.59459764953147e-06, | |
| "logits/chosen": -0.3965223431587219, | |
| "logits/rejected": -0.4247291684150696, | |
| "logps/chosen": -1.0226197242736816, | |
| "logps/rejected": -1.121930718421936, | |
| "loss": 1.0919, | |
| "odds_ratio_loss": 0.6925050616264343, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.1022619754076004, | |
| "rewards/margins": 0.00993109680712223, | |
| "rewards/rejected": -0.11219307035207748, | |
| "sft_loss": 1.0226197242736816, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.5690153367414981, | |
| "grad_norm": 0.5753230452537537, | |
| "learning_rate": 4.568803203605133e-06, | |
| "logits/chosen": -0.38987019658088684, | |
| "logits/rejected": -0.40249496698379517, | |
| "logps/chosen": -1.0238714218139648, | |
| "logps/rejected": -1.191584825515747, | |
| "loss": 1.0951, | |
| "odds_ratio_loss": 0.7120264768600464, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.10238714516162872, | |
| "rewards/margins": 0.016771327704191208, | |
| "rewards/rejected": -0.11915846914052963, | |
| "sft_loss": 1.0238714218139648, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.58679706601467, | |
| "grad_norm": 0.40169399976730347, | |
| "learning_rate": 4.542290396694462e-06, | |
| "logits/chosen": -0.4059433043003082, | |
| "logits/rejected": -0.4052697718143463, | |
| "logps/chosen": -0.9671312570571899, | |
| "logps/rejected": -1.0644605159759521, | |
| "loss": 1.0391, | |
| "odds_ratio_loss": 0.7196342349052429, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09671313315629959, | |
| "rewards/margins": 0.009732924401760101, | |
| "rewards/rejected": -0.1064460501074791, | |
| "sft_loss": 0.9671312570571899, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.6045787952878418, | |
| "grad_norm": 0.5619000792503357, | |
| "learning_rate": 4.515068434975298e-06, | |
| "logits/chosen": -0.4578043818473816, | |
| "logits/rejected": -0.4284750819206238, | |
| "logps/chosen": -0.9811161756515503, | |
| "logps/rejected": -1.1456761360168457, | |
| "loss": 1.0484, | |
| "odds_ratio_loss": 0.6727977991104126, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09811162203550339, | |
| "rewards/margins": 0.016455989331007004, | |
| "rewards/rejected": -0.11456761509180069, | |
| "sft_loss": 0.9811161756515503, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.6223605245610135, | |
| "grad_norm": 0.5821824073791504, | |
| "learning_rate": 4.487146770866887e-06, | |
| "logits/chosen": -0.34484004974365234, | |
| "logits/rejected": -0.3222612738609314, | |
| "logps/chosen": -1.0583232641220093, | |
| "logps/rejected": -1.117333173751831, | |
| "loss": 1.1304, | |
| "odds_ratio_loss": 0.7205663919448853, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": -0.10583231598138809, | |
| "rewards/margins": 0.005901001859456301, | |
| "rewards/rejected": -0.1117333322763443, | |
| "sft_loss": 1.0583232641220093, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.6401422538341853, | |
| "grad_norm": 0.28447961807250977, | |
| "learning_rate": 4.458535099749666e-06, | |
| "logits/chosen": -0.43229636549949646, | |
| "logits/rejected": -0.40540462732315063, | |
| "logps/chosen": -1.1308929920196533, | |
| "logps/rejected": -1.0958976745605469, | |
| "loss": 1.2174, | |
| "odds_ratio_loss": 0.8652679324150085, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.11308930814266205, | |
| "rewards/margins": -0.0034995335154235363, | |
| "rewards/rejected": -0.10958977788686752, | |
| "sft_loss": 1.1308929920196533, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.6579239831073572, | |
| "grad_norm": 0.27178603410720825, | |
| "learning_rate": 4.429243356598694e-06, | |
| "logits/chosen": -0.40932542085647583, | |
| "logits/rejected": -0.3859841227531433, | |
| "logps/chosen": -0.9554696083068848, | |
| "logps/rejected": -1.1517064571380615, | |
| "loss": 1.0243, | |
| "odds_ratio_loss": 0.6880883574485779, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.09554696083068848, | |
| "rewards/margins": 0.019623693078756332, | |
| "rewards/rejected": -0.11517064273357391, | |
| "sft_loss": 0.9554696083068848, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.675705712380529, | |
| "grad_norm": 0.34544578194618225, | |
| "learning_rate": 4.399281712533875e-06, | |
| "logits/chosen": -0.32934245467185974, | |
| "logits/rejected": -0.3599315285682678, | |
| "logps/chosen": -0.9367265701293945, | |
| "logps/rejected": -1.0202996730804443, | |
| "loss": 1.0101, | |
| "odds_ratio_loss": 0.7333763837814331, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.09367264807224274, | |
| "rewards/margins": 0.008357317186892033, | |
| "rewards/rejected": -0.1020299643278122, | |
| "sft_loss": 0.9367265701293945, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.6934874416537008, | |
| "grad_norm": 0.48474597930908203, | |
| "learning_rate": 4.368660571288192e-06, | |
| "logits/chosen": -0.3377426266670227, | |
| "logits/rejected": -0.32565537095069885, | |
| "logps/chosen": -0.9353078007698059, | |
| "logps/rejected": -1.0242602825164795, | |
| "loss": 1.0071, | |
| "odds_ratio_loss": 0.7176766395568848, | |
| "rewards/accuracies": 0.45625001192092896, | |
| "rewards/chosen": -0.09353077411651611, | |
| "rewards/margins": 0.008895261213183403, | |
| "rewards/rejected": -0.10242603719234467, | |
| "sft_loss": 0.9353078007698059, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.7112691709268726, | |
| "grad_norm": 0.3825822174549103, | |
| "learning_rate": 4.337390565595163e-06, | |
| "logits/chosen": -0.4158423840999603, | |
| "logits/rejected": -0.36646509170532227, | |
| "logps/chosen": -1.0673354864120483, | |
| "logps/rejected": -1.0877690315246582, | |
| "loss": 1.1448, | |
| "odds_ratio_loss": 0.7746785879135132, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.1067335456609726, | |
| "rewards/margins": 0.0020433522295206785, | |
| "rewards/rejected": -0.10877690464258194, | |
| "sft_loss": 1.0673354864120483, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.7290509002000445, | |
| "grad_norm": 0.36279189586639404, | |
| "learning_rate": 4.305482553496786e-06, | |
| "logits/chosen": -0.33700472116470337, | |
| "logits/rejected": -0.3831488788127899, | |
| "logps/chosen": -0.9607623815536499, | |
| "logps/rejected": -1.0405422449111938, | |
| "loss": 1.0363, | |
| "odds_ratio_loss": 0.7554237842559814, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09607623517513275, | |
| "rewards/margins": 0.007977982982993126, | |
| "rewards/rejected": -0.10405422747135162, | |
| "sft_loss": 0.9607623815536499, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.7468326294732163, | |
| "grad_norm": 0.457087904214859, | |
| "learning_rate": 4.272947614573244e-06, | |
| "logits/chosen": -0.3999176621437073, | |
| "logits/rejected": -0.3756122291088104, | |
| "logps/chosen": -1.0111384391784668, | |
| "logps/rejected": -1.0757354497909546, | |
| "loss": 1.0826, | |
| "odds_ratio_loss": 0.7144282460212708, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.10111384093761444, | |
| "rewards/margins": 0.006459714379161596, | |
| "rewards/rejected": -0.10757355391979218, | |
| "sft_loss": 1.0111384391784668, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.7646143587463881, | |
| "grad_norm": 0.2605019509792328, | |
| "learning_rate": 4.23979704609569e-06, | |
| "logits/chosen": -0.36384835839271545, | |
| "logits/rejected": -0.34030967950820923, | |
| "logps/chosen": -0.9615520238876343, | |
| "logps/rejected": -1.0373448133468628, | |
| "loss": 1.0309, | |
| "odds_ratio_loss": 0.6935026049613953, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.09615520387887955, | |
| "rewards/margins": 0.007579285651445389, | |
| "rewards/rejected": -0.10373447835445404, | |
| "sft_loss": 0.9615520238876343, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.78239608801956, | |
| "grad_norm": 0.41911929845809937, | |
| "learning_rate": 4.206042359103435e-06, | |
| "logits/chosen": -0.38596296310424805, | |
| "logits/rejected": -0.37879234552383423, | |
| "logps/chosen": -0.9808257222175598, | |
| "logps/rejected": -1.121048927307129, | |
| "loss": 1.0531, | |
| "odds_ratio_loss": 0.7229377627372742, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.09808257222175598, | |
| "rewards/margins": 0.014022317714989185, | |
| "rewards/rejected": -0.11210489273071289, | |
| "sft_loss": 0.9808257222175598, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.8001778172927317, | |
| "grad_norm": 0.7460839748382568, | |
| "learning_rate": 4.17169527440691e-06, | |
| "logits/chosen": -0.39514169096946716, | |
| "logits/rejected": -0.3737938106060028, | |
| "logps/chosen": -0.9438737630844116, | |
| "logps/rejected": -1.0060594081878662, | |
| "loss": 1.0182, | |
| "odds_ratio_loss": 0.7436385154724121, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09438737481832504, | |
| "rewards/margins": 0.006218560039997101, | |
| "rewards/rejected": -0.10060594230890274, | |
| "sft_loss": 0.9438737630844116, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.8179595465659035, | |
| "grad_norm": 0.5300458669662476, | |
| "learning_rate": 4.136767718517797e-06, | |
| "logits/chosen": -0.3699805736541748, | |
| "logits/rejected": -0.3850511312484741, | |
| "logps/chosen": -0.959467887878418, | |
| "logps/rejected": -1.100988507270813, | |
| "loss": 1.0256, | |
| "odds_ratio_loss": 0.6614881753921509, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.0959467813372612, | |
| "rewards/margins": 0.014152060262858868, | |
| "rewards/rejected": -0.11009885370731354, | |
| "sft_loss": 0.959467887878418, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.8357412758390753, | |
| "grad_norm": 0.9485012292861938, | |
| "learning_rate": 4.1012718195077196e-06, | |
| "logits/chosen": -0.37103739380836487, | |
| "logits/rejected": -0.3039020895957947, | |
| "logps/chosen": -0.9647709131240845, | |
| "logps/rejected": -1.0279747247695923, | |
| "loss": 1.0376, | |
| "odds_ratio_loss": 0.7286756038665771, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": -0.09647707641124725, | |
| "rewards/margins": 0.006320389453321695, | |
| "rewards/rejected": -0.10279747098684311, | |
| "sft_loss": 0.9647709131240845, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.8535230051122472, | |
| "grad_norm": 0.5754956603050232, | |
| "learning_rate": 4.065219902796953e-06, | |
| "logits/chosen": -0.40020495653152466, | |
| "logits/rejected": -0.39535146951675415, | |
| "logps/chosen": -0.9706109166145325, | |
| "logps/rejected": -1.093976378440857, | |
| "loss": 1.0453, | |
| "odds_ratio_loss": 0.7464355230331421, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.09706110507249832, | |
| "rewards/margins": 0.01233654748648405, | |
| "rewards/rejected": -0.10939764976501465, | |
| "sft_loss": 0.9706109166145325, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.871304734385419, | |
| "grad_norm": 0.3195387125015259, | |
| "learning_rate": 4.028624486874608e-06, | |
| "logits/chosen": -0.4315417408943176, | |
| "logits/rejected": -0.36453911662101746, | |
| "logps/chosen": -0.9465911984443665, | |
| "logps/rejected": -1.1121985912322998, | |
| "loss": 1.0194, | |
| "odds_ratio_loss": 0.7276239991188049, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.09465911984443665, | |
| "rewards/margins": 0.016560742631554604, | |
| "rewards/rejected": -0.1112198606133461, | |
| "sft_loss": 0.9465911984443665, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.8890864636585908, | |
| "grad_norm": 0.6305994391441345, | |
| "learning_rate": 3.99149827895177e-06, | |
| "logits/chosen": -0.38445502519607544, | |
| "logits/rejected": -0.38218945264816284, | |
| "logps/chosen": -1.0171244144439697, | |
| "logps/rejected": -1.0506142377853394, | |
| "loss": 1.0913, | |
| "odds_ratio_loss": 0.7415187358856201, | |
| "rewards/accuracies": 0.4312500059604645, | |
| "rewards/chosen": -0.10171245038509369, | |
| "rewards/margins": 0.0033489768393337727, | |
| "rewards/rejected": -0.10506142675876617, | |
| "sft_loss": 1.0171244144439697, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.8890864636585908, | |
| "eval_logits/chosen": -0.34904247522354126, | |
| "eval_logits/rejected": -0.31755369901657104, | |
| "eval_logps/chosen": -0.9676439166069031, | |
| "eval_logps/rejected": -1.1074860095977783, | |
| "eval_loss": 1.0354068279266357, | |
| "eval_odds_ratio_loss": 0.6776295900344849, | |
| "eval_rewards/accuracies": 0.5180000066757202, | |
| "eval_rewards/chosen": -0.09676438570022583, | |
| "eval_rewards/margins": 0.013984210789203644, | |
| "eval_rewards/rejected": -0.11074860394001007, | |
| "eval_runtime": 185.9798, | |
| "eval_samples_per_second": 5.377, | |
| "eval_sft_loss": 0.9676439166069031, | |
| "eval_steps_per_second": 2.688, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.9068681929317626, | |
| "grad_norm": 0.33740749955177307, | |
| "learning_rate": 3.953854170549114e-06, | |
| "logits/chosen": -0.3074025809764862, | |
| "logits/rejected": -0.30263853073120117, | |
| "logps/chosen": -0.9824435114860535, | |
| "logps/rejected": -1.0204169750213623, | |
| "loss": 1.0555, | |
| "odds_ratio_loss": 0.7308207750320435, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.09824434667825699, | |
| "rewards/margins": 0.0037973597645759583, | |
| "rewards/rejected": -0.10204169899225235, | |
| "sft_loss": 0.9824435114860535, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.9246499222049345, | |
| "grad_norm": 0.4032406210899353, | |
| "learning_rate": 3.91570523302051e-06, | |
| "logits/chosen": -0.3414192199707031, | |
| "logits/rejected": -0.36243736743927, | |
| "logps/chosen": -0.8989545702934265, | |
| "logps/rejected": -1.0376076698303223, | |
| "loss": 0.9695, | |
| "odds_ratio_loss": 0.7055255174636841, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.08989545702934265, | |
| "rewards/margins": 0.013865319080650806, | |
| "rewards/rejected": -0.10376076400279999, | |
| "sft_loss": 0.8989545702934265, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.9424316514781063, | |
| "grad_norm": 0.3632182776927948, | |
| "learning_rate": 3.8770647130141996e-06, | |
| "logits/chosen": -0.3258126378059387, | |
| "logits/rejected": -0.33273980021476746, | |
| "logps/chosen": -0.9584708213806152, | |
| "logps/rejected": -1.0552600622177124, | |
| "loss": 1.0316, | |
| "odds_ratio_loss": 0.731722891330719, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.09584707766771317, | |
| "rewards/margins": 0.009678924456238747, | |
| "rewards/rejected": -0.10552600771188736, | |
| "sft_loss": 0.9584708213806152, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.960213380751278, | |
| "grad_norm": 0.3121795058250427, | |
| "learning_rate": 3.837946027873086e-06, | |
| "logits/chosen": -0.32046863436698914, | |
| "logits/rejected": -0.3653668463230133, | |
| "logps/chosen": -0.966636061668396, | |
| "logps/rejected": -1.1031057834625244, | |
| "loss": 1.0367, | |
| "odds_ratio_loss": 0.7007311582565308, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.09666360169649124, | |
| "rewards/margins": 0.01364696491509676, | |
| "rewards/rejected": -0.11031056940555573, | |
| "sft_loss": 0.966636061668396, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.9779951100244498, | |
| "grad_norm": 0.6487416625022888, | |
| "learning_rate": 3.7983627609757713e-06, | |
| "logits/chosen": -0.34747475385665894, | |
| "logits/rejected": -0.3490690290927887, | |
| "logps/chosen": -0.9615602493286133, | |
| "logps/rejected": -1.0271753072738647, | |
| "loss": 1.0318, | |
| "odds_ratio_loss": 0.702663779258728, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.09615601599216461, | |
| "rewards/margins": 0.0065615237690508366, | |
| "rewards/rejected": -0.10271754115819931, | |
| "sft_loss": 0.9615602493286133, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.9957768392976217, | |
| "grad_norm": 0.3890874683856964, | |
| "learning_rate": 3.758328657019924e-06, | |
| "logits/chosen": -0.37014687061309814, | |
| "logits/rejected": -0.4008961319923401, | |
| "logps/chosen": -0.9199098348617554, | |
| "logps/rejected": -1.0562833547592163, | |
| "loss": 0.9886, | |
| "odds_ratio_loss": 0.6868860721588135, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.09199099242687225, | |
| "rewards/margins": 0.013637351803481579, | |
| "rewards/rejected": -0.1056283488869667, | |
| "sft_loss": 0.9199098348617554, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 1.0135585685707935, | |
| "grad_norm": 1.5021965503692627, | |
| "learning_rate": 3.717857617249642e-06, | |
| "logits/chosen": -0.409252405166626, | |
| "logits/rejected": -0.3774147033691406, | |
| "logps/chosen": -1.0592560768127441, | |
| "logps/rejected": -1.1887257099151611, | |
| "loss": 1.135, | |
| "odds_ratio_loss": 0.7577823400497437, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.10592560470104218, | |
| "rewards/margins": 0.012946966104209423, | |
| "rewards/rejected": -0.11887258291244507, | |
| "sft_loss": 1.0592560768127441, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 1.0313402978439654, | |
| "grad_norm": 0.36601969599723816, | |
| "learning_rate": 3.6769636946284543e-06, | |
| "logits/chosen": -0.33855992555618286, | |
| "logits/rejected": -0.38329094648361206, | |
| "logps/chosen": -0.9246651530265808, | |
| "logps/rejected": -1.0259661674499512, | |
| "loss": 0.9949, | |
| "odds_ratio_loss": 0.7019587755203247, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09246651828289032, | |
| "rewards/margins": 0.01013010274618864, | |
| "rewards/rejected": -0.10259661823511124, | |
| "sft_loss": 0.9246651530265808, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 1.049122027117137, | |
| "grad_norm": 0.3644584119319916, | |
| "learning_rate": 3.6356610889596355e-06, | |
| "logits/chosen": -0.3362785577774048, | |
| "logits/rejected": -0.3195570707321167, | |
| "logps/chosen": -0.9757383465766907, | |
| "logps/rejected": -1.0168259143829346, | |
| "loss": 1.0499, | |
| "odds_ratio_loss": 0.7411800622940063, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09757383167743683, | |
| "rewards/margins": 0.004108763299882412, | |
| "rewards/rejected": -0.10168258845806122, | |
| "sft_loss": 0.9757383465766907, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 1.066903756390309, | |
| "grad_norm": 0.38790592551231384, | |
| "learning_rate": 3.593964141955541e-06, | |
| "logits/chosen": -0.31955039501190186, | |
| "logits/rejected": -0.3287174701690674, | |
| "logps/chosen": -0.9446002244949341, | |
| "logps/rejected": -0.9857986569404602, | |
| "loss": 1.0183, | |
| "odds_ratio_loss": 0.7368658185005188, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09446002542972565, | |
| "rewards/margins": 0.004119834862649441, | |
| "rewards/rejected": -0.09857985377311707, | |
| "sft_loss": 0.9446002244949341, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 1.0846854856634809, | |
| "grad_norm": 0.3323744237422943, | |
| "learning_rate": 3.5518873322576573e-06, | |
| "logits/chosen": -0.43425217270851135, | |
| "logits/rejected": -0.3568256199359894, | |
| "logps/chosen": -0.9986424446105957, | |
| "logps/rejected": -1.0531480312347412, | |
| "loss": 1.073, | |
| "odds_ratio_loss": 0.7439261674880981, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.09986423701047897, | |
| "rewards/margins": 0.005450558383017778, | |
| "rewards/rejected": -0.10531480610370636, | |
| "sft_loss": 0.9986424446105957, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 1.1024672149366526, | |
| "grad_norm": 0.45893725752830505, | |
| "learning_rate": 3.5094452704091143e-06, | |
| "logits/chosen": -0.3812747299671173, | |
| "logits/rejected": -0.36471351981163025, | |
| "logps/chosen": -0.9423580169677734, | |
| "logps/rejected": -1.0641114711761475, | |
| "loss": 1.0114, | |
| "odds_ratio_loss": 0.6907029747962952, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09423580020666122, | |
| "rewards/margins": 0.01217535138130188, | |
| "rewards/rejected": -0.10641114413738251, | |
| "sft_loss": 0.9423580169677734, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 1.1202489442098245, | |
| "grad_norm": 0.5117968916893005, | |
| "learning_rate": 3.46665269378139e-06, | |
| "logits/chosen": -0.3292369842529297, | |
| "logits/rejected": -0.3725055158138275, | |
| "logps/chosen": -0.9826286435127258, | |
| "logps/rejected": -1.0871622562408447, | |
| "loss": 1.0548, | |
| "odds_ratio_loss": 0.7213753461837769, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09826286137104034, | |
| "rewards/margins": 0.010453373193740845, | |
| "rewards/rejected": -0.10871622711420059, | |
| "sft_loss": 0.9826286435127258, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 1.1380306734829961, | |
| "grad_norm": 0.5622742176055908, | |
| "learning_rate": 3.4235244614569794e-06, | |
| "logits/chosen": -0.3315224051475525, | |
| "logits/rejected": -0.3257826566696167, | |
| "logps/chosen": -1.1072447299957275, | |
| "logps/rejected": -1.0443857908248901, | |
| "loss": 1.1924, | |
| "odds_ratio_loss": 0.8511736989021301, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.11072447150945663, | |
| "rewards/margins": -0.006285896059125662, | |
| "rewards/rejected": -0.10443858057260513, | |
| "sft_loss": 1.1072447299957275, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 1.155812402756168, | |
| "grad_norm": 0.27428311109542847, | |
| "learning_rate": 3.3800755490698008e-06, | |
| "logits/chosen": -0.30900219082832336, | |
| "logits/rejected": -0.33938735723495483, | |
| "logps/chosen": -0.9312244653701782, | |
| "logps/rejected": -1.0983222723007202, | |
| "loss": 0.9964, | |
| "odds_ratio_loss": 0.651997447013855, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.0931224375963211, | |
| "rewards/margins": 0.01670977845788002, | |
| "rewards/rejected": -0.10983221232891083, | |
| "sft_loss": 0.9312244653701782, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 1.17359413202934, | |
| "grad_norm": 1.0422977209091187, | |
| "learning_rate": 3.3363210436051287e-06, | |
| "logits/chosen": -0.3527902662754059, | |
| "logits/rejected": -0.3563137948513031, | |
| "logps/chosen": -0.978245735168457, | |
| "logps/rejected": -1.0940849781036377, | |
| "loss": 1.0514, | |
| "odds_ratio_loss": 0.73140949010849, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.0978245884180069, | |
| "rewards/margins": 0.011583918705582619, | |
| "rewards/rejected": -0.10940849781036377, | |
| "sft_loss": 0.978245735168457, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 1.1913758613025116, | |
| "grad_norm": 0.4168451428413391, | |
| "learning_rate": 3.292276138160867e-06, | |
| "logits/chosen": -0.28714054822921753, | |
| "logits/rejected": -0.30155253410339355, | |
| "logps/chosen": -0.934456467628479, | |
| "logps/rejected": -1.0636101961135864, | |
| "loss": 1.0032, | |
| "odds_ratio_loss": 0.6879295110702515, | |
| "rewards/accuracies": 0.4312500059604645, | |
| "rewards/chosen": -0.09344564378261566, | |
| "rewards/margins": 0.012915370985865593, | |
| "rewards/rejected": -0.1063610091805458, | |
| "sft_loss": 0.934456467628479, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 1.2091575905756835, | |
| "grad_norm": 0.34239086508750916, | |
| "learning_rate": 3.2479561266719694e-06, | |
| "logits/chosen": -0.381683886051178, | |
| "logits/rejected": -0.37388402223587036, | |
| "logps/chosen": -0.9762662649154663, | |
| "logps/rejected": -1.0414526462554932, | |
| "loss": 1.0493, | |
| "odds_ratio_loss": 0.7306024432182312, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09762662649154663, | |
| "rewards/margins": 0.006518647074699402, | |
| "rewards/rejected": -0.10414527356624603, | |
| "sft_loss": 0.9762662649154663, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 1.2269393198488552, | |
| "grad_norm": 0.4666767716407776, | |
| "learning_rate": 3.2033763985998533e-06, | |
| "logits/chosen": -0.3561275601387024, | |
| "logits/rejected": -0.3666972517967224, | |
| "logps/chosen": -0.9278993606567383, | |
| "logps/rejected": -1.172456979751587, | |
| "loss": 0.9924, | |
| "odds_ratio_loss": 0.6447319984436035, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.09278994053602219, | |
| "rewards/margins": 0.024455763399600983, | |
| "rewards/rejected": -0.11724568903446198, | |
| "sft_loss": 0.9278993606567383, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 1.244721049122027, | |
| "grad_norm": 0.4466889202594757, | |
| "learning_rate": 3.1585524335886335e-06, | |
| "logits/chosen": -0.3700794279575348, | |
| "logits/rejected": -0.37532711029052734, | |
| "logps/chosen": -0.893964409828186, | |
| "logps/rejected": -1.0242712497711182, | |
| "loss": 0.9628, | |
| "odds_ratio_loss": 0.6878638863563538, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08939644694328308, | |
| "rewards/margins": 0.013030675239861012, | |
| "rewards/rejected": -0.10242712497711182, | |
| "sft_loss": 0.893964409828186, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 1.262502778395199, | |
| "grad_norm": 0.6432116031646729, | |
| "learning_rate": 3.1134997960900536e-06, | |
| "logits/chosen": -0.3843459486961365, | |
| "logits/rejected": -0.4183478355407715, | |
| "logps/chosen": -0.8787266612052917, | |
| "logps/rejected": -1.1227346658706665, | |
| "loss": 0.9417, | |
| "odds_ratio_loss": 0.6295467615127563, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.08787266910076141, | |
| "rewards/margins": 0.024400796741247177, | |
| "rewards/rejected": -0.11227346956729889, | |
| "sft_loss": 0.8787266612052917, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 1.2802845076683709, | |
| "grad_norm": 0.47079232335090637, | |
| "learning_rate": 3.0682341299589583e-06, | |
| "logits/chosen": -0.3750189244747162, | |
| "logits/rejected": -0.33040302991867065, | |
| "logps/chosen": -0.9284566640853882, | |
| "logps/rejected": -0.9662970304489136, | |
| "loss": 1.0031, | |
| "odds_ratio_loss": 0.7467560172080994, | |
| "rewards/accuracies": 0.4124999940395355, | |
| "rewards/chosen": -0.09284567832946777, | |
| "rewards/margins": 0.0037840281147509813, | |
| "rewards/rejected": -0.09662970155477524, | |
| "sft_loss": 0.9284566640853882, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 1.2980662369415426, | |
| "grad_norm": 0.4881021976470947, | |
| "learning_rate": 3.022771153021201e-06, | |
| "logits/chosen": -0.3772386610507965, | |
| "logits/rejected": -0.3512099087238312, | |
| "logps/chosen": -0.9160524606704712, | |
| "logps/rejected": -1.0388538837432861, | |
| "loss": 0.986, | |
| "odds_ratio_loss": 0.6990936994552612, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.09160524606704712, | |
| "rewards/margins": 0.012280138209462166, | |
| "rewards/rejected": -0.10388537496328354, | |
| "sft_loss": 0.9160524606704712, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 1.3158479662147144, | |
| "grad_norm": 0.3279300034046173, | |
| "learning_rate": 2.9771266516158625e-06, | |
| "logits/chosen": -0.33211830258369446, | |
| "logits/rejected": -0.3039989471435547, | |
| "logps/chosen": -0.9333264231681824, | |
| "logps/rejected": -1.0419334173202515, | |
| "loss": 1.0054, | |
| "odds_ratio_loss": 0.72088623046875, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09333264082670212, | |
| "rewards/margins": 0.010860702954232693, | |
| "rewards/rejected": -0.10419335216283798, | |
| "sft_loss": 0.9333264231681824, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 1.3336296954878861, | |
| "grad_norm": 0.311788409948349, | |
| "learning_rate": 2.9313164751136802e-06, | |
| "logits/chosen": -0.3910767436027527, | |
| "logits/rejected": -0.36302170157432556, | |
| "logps/chosen": -0.9149459004402161, | |
| "logps/rejected": -1.0412867069244385, | |
| "loss": 0.9824, | |
| "odds_ratio_loss": 0.6748364567756653, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.0914945974946022, | |
| "rewards/margins": 0.01263406127691269, | |
| "rewards/rejected": -0.1041286438703537, | |
| "sft_loss": 0.9149459004402161, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 1.351411424761058, | |
| "grad_norm": 0.5009350180625916, | |
| "learning_rate": 2.8853565304135956e-06, | |
| "logits/chosen": -0.28646108508110046, | |
| "logits/rejected": -0.3241187632083893, | |
| "logps/chosen": -0.988601803779602, | |
| "logps/rejected": -1.0276473760604858, | |
| "loss": 1.0645, | |
| "odds_ratio_loss": 0.759224534034729, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.09886018931865692, | |
| "rewards/margins": 0.0039045563898980618, | |
| "rewards/rejected": -0.10276474803686142, | |
| "sft_loss": 0.988601803779602, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 1.36919315403423, | |
| "grad_norm": 0.5821639895439148, | |
| "learning_rate": 2.839262776419313e-06, | |
| "logits/chosen": -0.345294713973999, | |
| "logits/rejected": -0.34865519404411316, | |
| "logps/chosen": -0.9152688980102539, | |
| "logps/rejected": -1.12654709815979, | |
| "loss": 0.9828, | |
| "odds_ratio_loss": 0.6755408644676208, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09152691066265106, | |
| "rewards/margins": 0.021127816289663315, | |
| "rewards/rejected": -0.11265470832586288, | |
| "sft_loss": 0.9152688980102539, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 1.3869748833074016, | |
| "grad_norm": 0.39795824885368347, | |
| "learning_rate": 2.793051218497817e-06, | |
| "logits/chosen": -0.27542608976364136, | |
| "logits/rejected": -0.27257028222084045, | |
| "logps/chosen": -0.931863009929657, | |
| "logps/rejected": -0.9498918652534485, | |
| "loss": 1.0074, | |
| "odds_ratio_loss": 0.7550782561302185, | |
| "rewards/accuracies": 0.4375, | |
| "rewards/chosen": -0.09318631142377853, | |
| "rewards/margins": 0.0018028710037469864, | |
| "rewards/rejected": -0.09498917311429977, | |
| "sft_loss": 0.931863009929657, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 1.4047566125805735, | |
| "grad_norm": 0.37384262681007385, | |
| "learning_rate": 2.7467379029217437e-06, | |
| "logits/chosen": -0.34524422883987427, | |
| "logits/rejected": -0.36011195182800293, | |
| "logps/chosen": -0.9515836834907532, | |
| "logps/rejected": -1.0694557428359985, | |
| "loss": 1.0211, | |
| "odds_ratio_loss": 0.6952496767044067, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09515835344791412, | |
| "rewards/margins": 0.011787201277911663, | |
| "rewards/rejected": -0.10694557428359985, | |
| "sft_loss": 0.9515836834907532, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 1.4225383418537452, | |
| "grad_norm": 0.30680692195892334, | |
| "learning_rate": 2.7003389112975546e-06, | |
| "logits/chosen": -0.26400548219680786, | |
| "logits/rejected": -0.20824924111366272, | |
| "logps/chosen": -0.9995955228805542, | |
| "logps/rejected": -1.0734318494796753, | |
| "loss": 1.0721, | |
| "odds_ratio_loss": 0.7255308628082275, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.09995955973863602, | |
| "rewards/margins": 0.007383632007986307, | |
| "rewards/rejected": -0.10734319686889648, | |
| "sft_loss": 0.9995955228805542, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 1.440320071126917, | |
| "grad_norm": 0.7603825926780701, | |
| "learning_rate": 2.653870354981437e-06, | |
| "logits/chosen": -0.36708512902259827, | |
| "logits/rejected": -0.4067977964878082, | |
| "logps/chosen": -0.869776725769043, | |
| "logps/rejected": -0.9957377314567566, | |
| "loss": 0.9397, | |
| "odds_ratio_loss": 0.6991982460021973, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08697767555713654, | |
| "rewards/margins": 0.012596105225384235, | |
| "rewards/rejected": -0.0995737761259079, | |
| "sft_loss": 0.869776725769043, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 1.458101800400089, | |
| "grad_norm": 0.8572419881820679, | |
| "learning_rate": 2.6073483694848777e-06, | |
| "logits/chosen": -0.3313853442668915, | |
| "logits/rejected": -0.2504517734050751, | |
| "logps/chosen": -0.9180091619491577, | |
| "logps/rejected": -1.0551806688308716, | |
| "loss": 0.9865, | |
| "odds_ratio_loss": 0.6853691339492798, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.09180092811584473, | |
| "rewards/margins": 0.013717141933739185, | |
| "rewards/rejected": -0.10551806539297104, | |
| "sft_loss": 0.9180091619491577, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 1.4758835296732609, | |
| "grad_norm": 0.2907600700855255, | |
| "learning_rate": 2.560789108871847e-06, | |
| "logits/chosen": -0.35712695121765137, | |
| "logits/rejected": -0.34705477952957153, | |
| "logps/chosen": -0.9147292971611023, | |
| "logps/rejected": -1.1361644268035889, | |
| "loss": 0.9806, | |
| "odds_ratio_loss": 0.6587303280830383, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.09147293865680695, | |
| "rewards/margins": 0.02214350923895836, | |
| "rewards/rejected": -0.113616444170475, | |
| "sft_loss": 0.9147292971611023, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 1.4936652589464325, | |
| "grad_norm": 0.9957931637763977, | |
| "learning_rate": 2.514208740149544e-06, | |
| "logits/chosen": -0.38370782136917114, | |
| "logits/rejected": -0.372738778591156, | |
| "logps/chosen": -1.0301647186279297, | |
| "logps/rejected": -1.131388783454895, | |
| "loss": 1.1016, | |
| "odds_ratio_loss": 0.7141064405441284, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.10301647335290909, | |
| "rewards/margins": 0.010122401639819145, | |
| "rewards/rejected": -0.11313886940479279, | |
| "sft_loss": 1.0301647186279297, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 1.5114469882196042, | |
| "grad_norm": 0.3347834050655365, | |
| "learning_rate": 2.46762343765464e-06, | |
| "logits/chosen": -0.33272939920425415, | |
| "logits/rejected": -0.3354397416114807, | |
| "logps/chosen": -0.9821497797966003, | |
| "logps/rejected": -1.1356861591339111, | |
| "loss": 1.0494, | |
| "odds_ratio_loss": 0.672347903251648, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09821496903896332, | |
| "rewards/margins": 0.015353633090853691, | |
| "rewards/rejected": -0.11356861889362335, | |
| "sft_loss": 0.9821497797966003, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 1.5292287174927761, | |
| "grad_norm": 0.40781450271606445, | |
| "learning_rate": 2.4210493774369903e-06, | |
| "logits/chosen": -0.3659764528274536, | |
| "logits/rejected": -0.34343641996383667, | |
| "logps/chosen": -0.9932387471199036, | |
| "logps/rejected": -1.0735210180282593, | |
| "loss": 1.0663, | |
| "odds_ratio_loss": 0.7305824160575867, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09932386875152588, | |
| "rewards/margins": 0.008028226904571056, | |
| "rewards/rejected": -0.10735210031270981, | |
| "sft_loss": 0.9932387471199036, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 1.547010446765948, | |
| "grad_norm": 0.33270904421806335, | |
| "learning_rate": 2.374502731642732e-06, | |
| "logits/chosen": -0.33156028389930725, | |
| "logits/rejected": -0.3256151080131531, | |
| "logps/chosen": -0.9762036204338074, | |
| "logps/rejected": -1.0732605457305908, | |
| "loss": 1.0483, | |
| "odds_ratio_loss": 0.7209652662277222, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09762036800384521, | |
| "rewards/margins": 0.009705697186291218, | |
| "rewards/rejected": -0.10732606798410416, | |
| "sft_loss": 0.9762036204338074, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 1.56479217603912, | |
| "grad_norm": 0.46649253368377686, | |
| "learning_rate": 2.3279996628987556e-06, | |
| "logits/chosen": -0.3505496084690094, | |
| "logits/rejected": -0.3284318149089813, | |
| "logps/chosen": -0.9539216756820679, | |
| "logps/rejected": -1.0178234577178955, | |
| "loss": 1.0269, | |
| "odds_ratio_loss": 0.7295688390731812, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09539216756820679, | |
| "rewards/margins": 0.006390177644789219, | |
| "rewards/rejected": -0.10178234428167343, | |
| "sft_loss": 0.9539216756820679, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 1.5825739053122916, | |
| "grad_norm": 0.343382865190506, | |
| "learning_rate": 2.281556318700474e-06, | |
| "logits/chosen": -0.2859468460083008, | |
| "logits/rejected": -0.25978535413742065, | |
| "logps/chosen": -0.904071033000946, | |
| "logps/rejected": -0.9673022031784058, | |
| "loss": 0.9788, | |
| "odds_ratio_loss": 0.7473067045211792, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.0904071107506752, | |
| "rewards/margins": 0.00632312148809433, | |
| "rewards/rejected": -0.09673022478818893, | |
| "sft_loss": 0.904071033000946, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 1.6003556345854635, | |
| "grad_norm": 0.6206201314926147, | |
| "learning_rate": 2.2351888258048408e-06, | |
| "logits/chosen": -0.3074144423007965, | |
| "logits/rejected": -0.2645527720451355, | |
| "logps/chosen": -0.8916131854057312, | |
| "logps/rejected": -0.9986615180969238, | |
| "loss": 0.9603, | |
| "odds_ratio_loss": 0.6866299510002136, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.08916132152080536, | |
| "rewards/margins": 0.010704840533435345, | |
| "rewards/rejected": -0.09986615926027298, | |
| "sft_loss": 0.8916131854057312, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 1.6181373638586352, | |
| "grad_norm": 0.3601900339126587, | |
| "learning_rate": 2.188913284630584e-06, | |
| "logits/chosen": -0.33852243423461914, | |
| "logits/rejected": -0.3135743737220764, | |
| "logps/chosen": -0.9911006689071655, | |
| "logps/rejected": -1.016789197921753, | |
| "loss": 1.0679, | |
| "odds_ratio_loss": 0.7680201530456543, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.09911007434129715, | |
| "rewards/margins": 0.0025688547175377607, | |
| "rewards/rejected": -0.10167893022298813, | |
| "sft_loss": 0.9911006689071655, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 1.635919093131807, | |
| "grad_norm": 0.6057630777359009, | |
| "learning_rate": 2.1427457636675652e-06, | |
| "logits/chosen": -0.3320189118385315, | |
| "logits/rejected": -0.28204983472824097, | |
| "logps/chosen": -1.0480351448059082, | |
| "logps/rejected": -1.1421617269515991, | |
| "loss": 1.1202, | |
| "odds_ratio_loss": 0.7219060659408569, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.10480351746082306, | |
| "rewards/margins": 0.00941266119480133, | |
| "rewards/rejected": -0.11421617120504379, | |
| "sft_loss": 1.0480351448059082, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 1.653700822404979, | |
| "grad_norm": 0.27687886357307434, | |
| "learning_rate": 2.096702293897247e-06, | |
| "logits/chosen": -0.3558569550514221, | |
| "logits/rejected": -0.4100232720375061, | |
| "logps/chosen": -0.9075578451156616, | |
| "logps/rejected": -1.1192221641540527, | |
| "loss": 0.9773, | |
| "odds_ratio_loss": 0.6971360445022583, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.09075579047203064, | |
| "rewards/margins": 0.02116643264889717, | |
| "rewards/rejected": -0.11192221939563751, | |
| "sft_loss": 0.9075578451156616, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 1.6714825516781509, | |
| "grad_norm": 0.5104541182518005, | |
| "learning_rate": 2.0507988632261672e-06, | |
| "logits/chosen": -0.37269848585128784, | |
| "logits/rejected": -0.3488038182258606, | |
| "logps/chosen": -0.8780601620674133, | |
| "logps/rejected": -1.035788893699646, | |
| "loss": 0.9453, | |
| "odds_ratio_loss": 0.6724425554275513, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08780601620674133, | |
| "rewards/margins": 0.015772882848978043, | |
| "rewards/rejected": -0.10357888787984848, | |
| "sft_loss": 0.8780601620674133, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 1.6892642809513225, | |
| "grad_norm": 1.108080506324768, | |
| "learning_rate": 2.005051410934382e-06, | |
| "logits/chosen": -0.3843027949333191, | |
| "logits/rejected": -0.36695486307144165, | |
| "logps/chosen": -1.0294411182403564, | |
| "logps/rejected": -1.073974847793579, | |
| "loss": 1.1057, | |
| "odds_ratio_loss": 0.7625271081924438, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.10294412076473236, | |
| "rewards/margins": 0.004453369881957769, | |
| "rewards/rejected": -0.10739749670028687, | |
| "sft_loss": 1.0294411182403564, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 1.7070460102244942, | |
| "grad_norm": 0.6668155789375305, | |
| "learning_rate": 1.9594758221407843e-06, | |
| "logits/chosen": -0.30207034945487976, | |
| "logits/rejected": -0.31365981698036194, | |
| "logps/chosen": -0.8924224972724915, | |
| "logps/rejected": -1.0662165880203247, | |
| "loss": 0.9564, | |
| "odds_ratio_loss": 0.6395965218544006, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08924224227666855, | |
| "rewards/margins": 0.017379416152834892, | |
| "rewards/rejected": -0.10662166774272919, | |
| "sft_loss": 0.8924224972724915, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 1.724827739497666, | |
| "grad_norm": 0.5297231674194336, | |
| "learning_rate": 1.9140879222872408e-06, | |
| "logits/chosen": -0.3790926933288574, | |
| "logits/rejected": -0.34034663438796997, | |
| "logps/chosen": -0.9109382629394531, | |
| "logps/rejected": -0.9725145101547241, | |
| "loss": 0.9864, | |
| "odds_ratio_loss": 0.7550500631332397, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.09109383821487427, | |
| "rewards/margins": 0.006157620809972286, | |
| "rewards/rejected": -0.09725145250558853, | |
| "sft_loss": 0.9109382629394531, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 1.742609468770838, | |
| "grad_norm": 0.2978646457195282, | |
| "learning_rate": 1.8689034716434346e-06, | |
| "logits/chosen": -0.3594937026500702, | |
| "logits/rejected": -0.3786514699459076, | |
| "logps/chosen": -0.9791936874389648, | |
| "logps/rejected": -1.0208795070648193, | |
| "loss": 1.054, | |
| "odds_ratio_loss": 0.7475694417953491, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.09791935980319977, | |
| "rewards/margins": 0.0041685826145112514, | |
| "rewards/rejected": -0.10208795219659805, | |
| "sft_loss": 0.9791936874389648, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 1.76039119804401, | |
| "grad_norm": 0.3484848439693451, | |
| "learning_rate": 1.8239381598343576e-06, | |
| "logits/chosen": -0.29449883103370667, | |
| "logits/rejected": -0.3054262697696686, | |
| "logps/chosen": -0.9115015864372253, | |
| "logps/rejected": -1.0031999349594116, | |
| "loss": 0.9826, | |
| "odds_ratio_loss": 0.7106297016143799, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.09115016460418701, | |
| "rewards/margins": 0.00916983187198639, | |
| "rewards/rejected": -0.1003199964761734, | |
| "sft_loss": 0.9115015864372253, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 1.7781729273171816, | |
| "grad_norm": 2.2374985218048096, | |
| "learning_rate": 1.779207600392312e-06, | |
| "logits/chosen": -0.2810733914375305, | |
| "logits/rejected": -0.27120235562324524, | |
| "logps/chosen": -0.9607506990432739, | |
| "logps/rejected": -1.0408788919448853, | |
| "loss": 1.0328, | |
| "odds_ratio_loss": 0.7200591564178467, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.09607508033514023, | |
| "rewards/margins": 0.008012807928025723, | |
| "rewards/rejected": -0.10408788919448853, | |
| "sft_loss": 0.9607506990432739, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.7781729273171816, | |
| "eval_logits/chosen": -0.33078742027282715, | |
| "eval_logits/rejected": -0.29791274666786194, | |
| "eval_logps/chosen": -0.9451074004173279, | |
| "eval_logps/rejected": -1.0856181383132935, | |
| "eval_loss": 1.0125839710235596, | |
| "eval_odds_ratio_loss": 0.6747645735740662, | |
| "eval_rewards/accuracies": 0.515999972820282, | |
| "eval_rewards/chosen": -0.0945107489824295, | |
| "eval_rewards/margins": 0.014051074162125587, | |
| "eval_rewards/rejected": -0.10856182873249054, | |
| "eval_runtime": 185.8537, | |
| "eval_samples_per_second": 5.381, | |
| "eval_sft_loss": 0.9451074004173279, | |
| "eval_steps_per_second": 2.69, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.7959546565903532, | |
| "grad_norm": 0.7795166373252869, | |
| "learning_rate": 1.7347273253353552e-06, | |
| "logits/chosen": -0.33356940746307373, | |
| "logits/rejected": -0.3380289077758789, | |
| "logps/chosen": -0.918900191783905, | |
| "logps/rejected": -0.9768841862678528, | |
| "loss": 0.9932, | |
| "odds_ratio_loss": 0.7426038980484009, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09189002215862274, | |
| "rewards/margins": 0.005798395723104477, | |
| "rewards/rejected": -0.09768841415643692, | |
| "sft_loss": 0.918900191783905, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 1.8137363858635251, | |
| "grad_norm": 0.8157365322113037, | |
| "learning_rate": 1.690512779774029e-06, | |
| "logits/chosen": -0.3094736635684967, | |
| "logits/rejected": -0.28969138860702515, | |
| "logps/chosen": -0.9715908765792847, | |
| "logps/rejected": -1.1499989032745361, | |
| "loss": 1.037, | |
| "odds_ratio_loss": 0.6542772054672241, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.09715909510850906, | |
| "rewards/margins": 0.017840798944234848, | |
| "rewards/rejected": -0.11499989032745361, | |
| "sft_loss": 0.9715908765792847, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 1.831518115136697, | |
| "grad_norm": 0.5331993103027344, | |
| "learning_rate": 1.6465793165482838e-06, | |
| "logits/chosen": -0.274508535861969, | |
| "logits/rejected": -0.26048415899276733, | |
| "logps/chosen": -0.9679173231124878, | |
| "logps/rejected": -1.0533314943313599, | |
| "loss": 1.0376, | |
| "odds_ratio_loss": 0.6963869333267212, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.09679173678159714, | |
| "rewards/margins": 0.008541420102119446, | |
| "rewards/rejected": -0.10533314943313599, | |
| "sft_loss": 0.9679173231124878, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 1.849299844409869, | |
| "grad_norm": 0.4930827021598816, | |
| "learning_rate": 1.6029421908964305e-06, | |
| "logits/chosen": -0.3850288391113281, | |
| "logits/rejected": -0.3791029155254364, | |
| "logps/chosen": -0.8834483027458191, | |
| "logps/rejected": -1.2469079494476318, | |
| "loss": 0.9502, | |
| "odds_ratio_loss": 0.6672720313072205, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08834483474493027, | |
| "rewards/margins": 0.03634597733616829, | |
| "rewards/rejected": -0.12469079345464706, | |
| "sft_loss": 0.8834483027458191, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 1.8670815736830408, | |
| "grad_norm": 0.7664922475814819, | |
| "learning_rate": 1.559616555157985e-06, | |
| "logits/chosen": -0.30128011107444763, | |
| "logits/rejected": -0.33186617493629456, | |
| "logps/chosen": -0.9356236457824707, | |
| "logps/rejected": -1.047398328781128, | |
| "loss": 1.0066, | |
| "odds_ratio_loss": 0.7096288800239563, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.09356234967708588, | |
| "rewards/margins": 0.01117746438831091, | |
| "rewards/rejected": -0.10473982989788055, | |
| "sft_loss": 0.9356236457824707, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 1.8848633029562125, | |
| "grad_norm": 0.465348482131958, | |
| "learning_rate": 1.516617453512252e-06, | |
| "logits/chosen": -0.36206910014152527, | |
| "logits/rejected": -0.34239286184310913, | |
| "logps/chosen": -0.9592390060424805, | |
| "logps/rejected": -1.0232237577438354, | |
| "loss": 1.0338, | |
| "odds_ratio_loss": 0.7456762194633484, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.09592391550540924, | |
| "rewards/margins": 0.006398468278348446, | |
| "rewards/rejected": -0.10232237726449966, | |
| "sft_loss": 0.9592390060424805, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 1.9026450322293842, | |
| "grad_norm": 0.830959677696228, | |
| "learning_rate": 1.473959816754449e-06, | |
| "logits/chosen": -0.39980772137641907, | |
| "logits/rejected": -0.3537663221359253, | |
| "logps/chosen": -0.920127272605896, | |
| "logps/rejected": -0.9525257349014282, | |
| "loss": 0.9942, | |
| "odds_ratio_loss": 0.7409034967422485, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.09201272577047348, | |
| "rewards/margins": 0.0032398372422903776, | |
| "rewards/rejected": -0.09525256603956223, | |
| "sft_loss": 0.920127272605896, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 1.920426761502556, | |
| "grad_norm": 0.442227303981781, | |
| "learning_rate": 1.4316584571112213e-06, | |
| "logits/chosen": -0.23950842022895813, | |
| "logits/rejected": -0.25979962944984436, | |
| "logps/chosen": -0.9493446350097656, | |
| "logps/rejected": -1.02411687374115, | |
| "loss": 1.022, | |
| "odds_ratio_loss": 0.7267680764198303, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.09493447840213776, | |
| "rewards/margins": 0.007477219216525555, | |
| "rewards/rejected": -0.10241168737411499, | |
| "sft_loss": 0.9493446350097656, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 1.938208490775728, | |
| "grad_norm": 0.4206017851829529, | |
| "learning_rate": 1.389728063097306e-06, | |
| "logits/chosen": -0.23708462715148926, | |
| "logits/rejected": -0.24299781024456024, | |
| "logps/chosen": -0.9439695477485657, | |
| "logps/rejected": -1.1116364002227783, | |
| "loss": 1.0118, | |
| "odds_ratio_loss": 0.6782708764076233, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09439694881439209, | |
| "rewards/margins": 0.016766689717769623, | |
| "rewards/rejected": -0.1111636534333229, | |
| "sft_loss": 0.9439695477485657, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 1.9559902200488999, | |
| "grad_norm": 0.3826051354408264, | |
| "learning_rate": 1.348183194415179e-06, | |
| "logits/chosen": -0.332774817943573, | |
| "logits/rejected": -0.35824882984161377, | |
| "logps/chosen": -0.9340184926986694, | |
| "logps/rejected": -1.110877275466919, | |
| "loss": 1.0005, | |
| "odds_ratio_loss": 0.6648778915405273, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.09340184926986694, | |
| "rewards/margins": 0.01768588088452816, | |
| "rewards/rejected": -0.11108773946762085, | |
| "sft_loss": 0.9340184926986694, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 1.9737719493220716, | |
| "grad_norm": 0.3005673587322235, | |
| "learning_rate": 1.3070382768994015e-06, | |
| "logits/chosen": -0.30200204253196716, | |
| "logits/rejected": -0.3130107522010803, | |
| "logps/chosen": -0.9192419052124023, | |
| "logps/rejected": -0.9889400601387024, | |
| "loss": 0.9898, | |
| "odds_ratio_loss": 0.7055012583732605, | |
| "rewards/accuracies": 0.45625001192092896, | |
| "rewards/chosen": -0.09192419052124023, | |
| "rewards/margins": 0.006969820708036423, | |
| "rewards/rejected": -0.09889401495456696, | |
| "sft_loss": 0.9192419052124023, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 1.9915536785952432, | |
| "grad_norm": 0.4379596710205078, | |
| "learning_rate": 1.2663075975074746e-06, | |
| "logits/chosen": -0.3314594626426697, | |
| "logits/rejected": -0.33315131068229675, | |
| "logps/chosen": -0.9054539799690247, | |
| "logps/rejected": -1.0939247608184814, | |
| "loss": 0.9734, | |
| "odds_ratio_loss": 0.6797955632209778, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.0905454009771347, | |
| "rewards/margins": 0.018847089260816574, | |
| "rewards/rejected": -0.10939247906208038, | |
| "sft_loss": 0.9054539799690247, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 2.009335407868415, | |
| "grad_norm": 0.6127385497093201, | |
| "learning_rate": 1.2260052993589034e-06, | |
| "logits/chosen": -0.382732093334198, | |
| "logits/rejected": -0.36521822214126587, | |
| "logps/chosen": -1.0369594097137451, | |
| "logps/rejected": -1.0331060886383057, | |
| "loss": 1.1183, | |
| "odds_ratio_loss": 0.8130975961685181, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.10369595140218735, | |
| "rewards/margins": -0.0003853384405374527, | |
| "rewards/rejected": -0.10331060737371445, | |
| "sft_loss": 1.0369594097137451, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 2.027117137141587, | |
| "grad_norm": 0.3373187780380249, | |
| "learning_rate": 1.1861453768242099e-06, | |
| "logits/chosen": -0.3635232448577881, | |
| "logits/rejected": -0.3613505959510803, | |
| "logps/chosen": -0.9056431651115417, | |
| "logps/rejected": -1.0306495428085327, | |
| "loss": 0.9749, | |
| "odds_ratio_loss": 0.6926708221435547, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.0905643105506897, | |
| "rewards/margins": 0.012500641867518425, | |
| "rewards/rejected": -0.10306496918201447, | |
| "sft_loss": 0.9056431651115417, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 2.044898866414759, | |
| "grad_norm": 0.9102166891098022, | |
| "learning_rate": 1.1467416706655982e-06, | |
| "logits/chosen": -0.2888937294483185, | |
| "logits/rejected": -0.26064902544021606, | |
| "logps/chosen": -0.9796838760375977, | |
| "logps/rejected": -1.1222679615020752, | |
| "loss": 1.0522, | |
| "odds_ratio_loss": 0.7250452637672424, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.09796838462352753, | |
| "rewards/margins": 0.014258405193686485, | |
| "rewards/rejected": -0.11222679913043976, | |
| "sft_loss": 0.9796838760375977, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 2.062680595687931, | |
| "grad_norm": 0.3294011652469635, | |
| "learning_rate": 1.1078078632309559e-06, | |
| "logits/chosen": -0.34561508893966675, | |
| "logits/rejected": -0.3147248923778534, | |
| "logps/chosen": -0.9134725332260132, | |
| "logps/rejected": -1.0285111665725708, | |
| "loss": 0.9808, | |
| "odds_ratio_loss": 0.6730369329452515, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.09134725481271744, | |
| "rewards/margins": 0.011503859423100948, | |
| "rewards/rejected": -0.10285113006830215, | |
| "sft_loss": 0.9134725332260132, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 2.0804623249611023, | |
| "grad_norm": 0.34308087825775146, | |
| "learning_rate": 1.0693574737028627e-06, | |
| "logits/chosen": -0.3372167944908142, | |
| "logits/rejected": -0.33946290612220764, | |
| "logps/chosen": -0.9201191067695618, | |
| "logps/rejected": -1.0031434297561646, | |
| "loss": 0.9946, | |
| "odds_ratio_loss": 0.744364321231842, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.09201192110776901, | |
| "rewards/margins": 0.008302421309053898, | |
| "rewards/rejected": -0.10031434148550034, | |
| "sft_loss": 0.9201191067695618, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 2.098244054234274, | |
| "grad_norm": 0.5865955948829651, | |
| "learning_rate": 1.0314038534042586e-06, | |
| "logits/chosen": -0.2901017963886261, | |
| "logits/rejected": -0.32853323221206665, | |
| "logps/chosen": -0.9257968068122864, | |
| "logps/rejected": -1.0451035499572754, | |
| "loss": 0.9964, | |
| "odds_ratio_loss": 0.7055808901786804, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0925796777009964, | |
| "rewards/margins": 0.01193068828433752, | |
| "rewards/rejected": -0.10451038181781769, | |
| "sft_loss": 0.9257968068122864, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 2.116025783507446, | |
| "grad_norm": 0.41964584589004517, | |
| "learning_rate": 9.939601811623946e-07, | |
| "logits/chosen": -0.31542712450027466, | |
| "logits/rejected": -0.30006498098373413, | |
| "logps/chosen": -0.9362471699714661, | |
| "logps/rejected": -1.0245290994644165, | |
| "loss": 1.0084, | |
| "odds_ratio_loss": 0.7219125032424927, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09362472593784332, | |
| "rewards/margins": 0.008828198537230492, | |
| "rewards/rejected": -0.10245291888713837, | |
| "sft_loss": 0.9362471699714661, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 2.133807512780618, | |
| "grad_norm": 0.48077794909477234, | |
| "learning_rate": 9.570394587326825e-07, | |
| "logits/chosen": -0.29744619131088257, | |
| "logits/rejected": -0.34743356704711914, | |
| "logps/chosen": -0.9422229528427124, | |
| "logps/rejected": -1.1074718236923218, | |
| "loss": 1.0093, | |
| "odds_ratio_loss": 0.6704057455062866, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.094222292304039, | |
| "rewards/margins": 0.016524888575077057, | |
| "rewards/rejected": -0.11074719578027725, | |
| "sft_loss": 0.9422229528427124, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 2.15158924205379, | |
| "grad_norm": 0.3064732253551483, | |
| "learning_rate": 9.206545062840302e-07, | |
| "logits/chosen": -0.2666998505592346, | |
| "logits/rejected": -0.3201262652873993, | |
| "logps/chosen": -0.8927067518234253, | |
| "logps/rejected": -1.0634257793426514, | |
| "loss": 0.9575, | |
| "odds_ratio_loss": 0.6478100419044495, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08927067369222641, | |
| "rewards/margins": 0.017071900889277458, | |
| "rewards/rejected": -0.10634257644414902, | |
| "sft_loss": 0.8927067518234253, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 2.1693709713269618, | |
| "grad_norm": 0.3534330725669861, | |
| "learning_rate": 8.848179579472285e-07, | |
| "logits/chosen": -0.3102249801158905, | |
| "logits/rejected": -0.2955402433872223, | |
| "logps/chosen": -0.9082851409912109, | |
| "logps/rejected": -0.9553133845329285, | |
| "loss": 0.9795, | |
| "odds_ratio_loss": 0.7121320962905884, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09082850813865662, | |
| "rewards/margins": 0.00470283068716526, | |
| "rewards/rejected": -0.09553134441375732, | |
| "sft_loss": 0.9082851409912109, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 2.1871527006001332, | |
| "grad_norm": 0.6444931626319885, | |
| "learning_rate": 8.495422574279403e-07, | |
| "logits/chosen": -0.3936762809753418, | |
| "logits/rejected": -0.42016810178756714, | |
| "logps/chosen": -0.8496967554092407, | |
| "logps/rejected": -1.0362155437469482, | |
| "loss": 0.9135, | |
| "odds_ratio_loss": 0.6377807855606079, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08496967703104019, | |
| "rewards/margins": 0.018651869148015976, | |
| "rewards/rejected": -0.10362155735492706, | |
| "sft_loss": 0.8496967554092407, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 2.204934429873305, | |
| "grad_norm": 0.4805600941181183, | |
| "learning_rate": 8.148396536858063e-07, | |
| "logits/chosen": -0.3237206041812897, | |
| "logits/rejected": -0.3143185079097748, | |
| "logps/chosen": -0.9960983991622925, | |
| "logps/rejected": -1.1420572996139526, | |
| "loss": 1.0672, | |
| "odds_ratio_loss": 0.7113397121429443, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.0996098443865776, | |
| "rewards/margins": 0.014595886692404747, | |
| "rewards/rejected": -0.1142057403922081, | |
| "sft_loss": 0.9960983991622925, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 2.222716159146477, | |
| "grad_norm": 0.676315188407898, | |
| "learning_rate": 7.807221966811815e-07, | |
| "logits/chosen": -0.29545170068740845, | |
| "logits/rejected": -0.31817343831062317, | |
| "logps/chosen": -0.9420124292373657, | |
| "logps/rejected": -1.0276824235916138, | |
| "loss": 1.0181, | |
| "odds_ratio_loss": 0.7609573006629944, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.0942012369632721, | |
| "rewards/margins": 0.008567007258534431, | |
| "rewards/rejected": -0.10276825726032257, | |
| "sft_loss": 0.9420124292373657, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 2.240497888419649, | |
| "grad_norm": 0.3943430781364441, | |
| "learning_rate": 7.47201733190962e-07, | |
| "logits/chosen": -0.3520922362804413, | |
| "logits/rejected": -0.3318483829498291, | |
| "logps/chosen": -0.8970060348510742, | |
| "logps/rejected": -0.9855879545211792, | |
| "loss": 0.9669, | |
| "odds_ratio_loss": 0.6993352174758911, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0897006094455719, | |
| "rewards/margins": 0.008858194574713707, | |
| "rewards/rejected": -0.09855880588293076, | |
| "sft_loss": 0.8970060348510742, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 2.258279617692821, | |
| "grad_norm": 0.5184921026229858, | |
| "learning_rate": 7.142899026949721e-07, | |
| "logits/chosen": -0.33211636543273926, | |
| "logits/rejected": -0.3313821256160736, | |
| "logps/chosen": -0.9101552963256836, | |
| "logps/rejected": -0.9938360452651978, | |
| "loss": 0.9798, | |
| "odds_ratio_loss": 0.6968866586685181, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.09101552516222, | |
| "rewards/margins": 0.00836807768791914, | |
| "rewards/rejected": -0.09938360750675201, | |
| "sft_loss": 0.9101552963256836, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 2.2760613469659923, | |
| "grad_norm": 1.8007909059524536, | |
| "learning_rate": 6.819981333343273e-07, | |
| "logits/chosen": -0.3704894185066223, | |
| "logits/rejected": -0.3426709771156311, | |
| "logps/chosen": -0.9317655563354492, | |
| "logps/rejected": -1.0302845239639282, | |
| "loss": 1.003, | |
| "odds_ratio_loss": 0.7128146886825562, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.09317655861377716, | |
| "rewards/margins": 0.009851890616118908, | |
| "rewards/rejected": -0.10302845388650894, | |
| "sft_loss": 0.9317655563354492, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 2.293843076239164, | |
| "grad_norm": 0.4554091989994049, | |
| "learning_rate": 6.503376379431839e-07, | |
| "logits/chosen": -0.2947995066642761, | |
| "logits/rejected": -0.279682457447052, | |
| "logps/chosen": -0.9925037622451782, | |
| "logps/rejected": -0.9870964884757996, | |
| "loss": 1.068, | |
| "odds_ratio_loss": 0.7550127506256104, | |
| "rewards/accuracies": 0.41874998807907104, | |
| "rewards/chosen": -0.09925039112567902, | |
| "rewards/margins": -0.0005407325807027519, | |
| "rewards/rejected": -0.09870964288711548, | |
| "sft_loss": 0.9925037622451782, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 2.311624805512336, | |
| "grad_norm": 1.7697697877883911, | |
| "learning_rate": 6.193194101552502e-07, | |
| "logits/chosen": -0.31604236364364624, | |
| "logits/rejected": -0.35974448919296265, | |
| "logps/chosen": -0.936480700969696, | |
| "logps/rejected": -1.0702247619628906, | |
| "loss": 1.002, | |
| "odds_ratio_loss": 0.655421793460846, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.09364806860685349, | |
| "rewards/margins": 0.013374416157603264, | |
| "rewards/rejected": -0.1070224866271019, | |
| "sft_loss": 0.936480700969696, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 2.329406534785508, | |
| "grad_norm": 0.6282922625541687, | |
| "learning_rate": 5.889542205864083e-07, | |
| "logits/chosen": -0.3355167806148529, | |
| "logits/rejected": -0.3377595543861389, | |
| "logps/chosen": -0.9515066146850586, | |
| "logps/rejected": -1.0681602954864502, | |
| "loss": 1.0205, | |
| "odds_ratio_loss": 0.6903635859489441, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.0951506644487381, | |
| "rewards/margins": 0.01166537031531334, | |
| "rewards/rejected": -0.10681603848934174, | |
| "sft_loss": 0.9515066146850586, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 2.34718826405868, | |
| "grad_norm": 0.3864741027355194, | |
| "learning_rate": 5.592526130947862e-07, | |
| "logits/chosen": -0.31521058082580566, | |
| "logits/rejected": -0.3186022937297821, | |
| "logps/chosen": -0.9329264760017395, | |
| "logps/rejected": -1.0726194381713867, | |
| "loss": 1.0056, | |
| "odds_ratio_loss": 0.7264095544815063, | |
| "rewards/accuracies": 0.45625001192092896, | |
| "rewards/chosen": -0.09329266101121902, | |
| "rewards/margins": 0.013969297520816326, | |
| "rewards/rejected": -0.10726194083690643, | |
| "sft_loss": 0.9329264760017395, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 2.3649699933318518, | |
| "grad_norm": 0.8674092292785645, | |
| "learning_rate": 5.302249011195507e-07, | |
| "logits/chosen": -0.3717043995857239, | |
| "logits/rejected": -0.3457496166229248, | |
| "logps/chosen": -0.9407739639282227, | |
| "logps/rejected": -0.9671589136123657, | |
| "loss": 1.015, | |
| "odds_ratio_loss": 0.7421091198921204, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09407740086317062, | |
| "rewards/margins": 0.0026384838856756687, | |
| "rewards/rejected": -0.09671588987112045, | |
| "sft_loss": 0.9407739639282227, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 2.382751722605023, | |
| "grad_norm": 0.8201255798339844, | |
| "learning_rate": 5.018811640997307e-07, | |
| "logits/chosen": -0.3262820839881897, | |
| "logits/rejected": -0.28208276629447937, | |
| "logps/chosen": -0.9741110801696777, | |
| "logps/rejected": -1.1972548961639404, | |
| "loss": 1.0409, | |
| "odds_ratio_loss": 0.6679055690765381, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.0974111258983612, | |
| "rewards/margins": 0.022314375266432762, | |
| "rewards/rejected": -0.11972548812627792, | |
| "sft_loss": 0.9741110801696777, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 2.400533451878195, | |
| "grad_norm": 0.3292596638202667, | |
| "learning_rate": 4.7423124397427105e-07, | |
| "logits/chosen": -0.37047189474105835, | |
| "logits/rejected": -0.31794866919517517, | |
| "logps/chosen": -0.9531441926956177, | |
| "logps/rejected": -1.015749216079712, | |
| "loss": 1.0256, | |
| "odds_ratio_loss": 0.7250458002090454, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09531442075967789, | |
| "rewards/margins": 0.006260508205741644, | |
| "rewards/rejected": -0.10157492011785507, | |
| "sft_loss": 0.9531441926956177, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 2.418315181151367, | |
| "grad_norm": 0.4776778817176819, | |
| "learning_rate": 4.472847417645787e-07, | |
| "logits/chosen": -0.2806258201599121, | |
| "logits/rejected": -0.3024401366710663, | |
| "logps/chosen": -0.9200853109359741, | |
| "logps/rejected": -1.114600419998169, | |
| "loss": 0.9877, | |
| "odds_ratio_loss": 0.6760807633399963, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.09200852364301682, | |
| "rewards/margins": 0.01945151947438717, | |
| "rewards/rejected": -0.11146005243062973, | |
| "sft_loss": 0.9200853109359741, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 2.436096910424539, | |
| "grad_norm": 0.3043542802333832, | |
| "learning_rate": 4.210510142406993e-07, | |
| "logits/chosen": -0.32727354764938354, | |
| "logits/rejected": -0.3754233717918396, | |
| "logps/chosen": -0.9101996421813965, | |
| "logps/rejected": -1.0942609310150146, | |
| "loss": 0.977, | |
| "odds_ratio_loss": 0.6675896644592285, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09101996570825577, | |
| "rewards/margins": 0.018406113609671593, | |
| "rewards/rejected": -0.10942608118057251, | |
| "sft_loss": 0.9101996421813965, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 2.4538786396977104, | |
| "grad_norm": 0.4151700437068939, | |
| "learning_rate": 3.9553917067232966e-07, | |
| "logits/chosen": -0.33969706296920776, | |
| "logits/rejected": -0.36881956458091736, | |
| "logps/chosen": -0.9399350881576538, | |
| "logps/rejected": -1.071777105331421, | |
| "loss": 1.0133, | |
| "odds_ratio_loss": 0.7333552241325378, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09399349987506866, | |
| "rewards/margins": 0.013184216804802418, | |
| "rewards/rejected": -0.10717771202325821, | |
| "sft_loss": 0.9399350881576538, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 2.4716603689708823, | |
| "grad_norm": 0.4568045437335968, | |
| "learning_rate": 3.707580696657509e-07, | |
| "logits/chosen": -0.2799975275993347, | |
| "logits/rejected": -0.30841827392578125, | |
| "logps/chosen": -0.9116710424423218, | |
| "logps/rejected": -0.9513956308364868, | |
| "loss": 0.9844, | |
| "odds_ratio_loss": 0.7269908785820007, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.09116710722446442, | |
| "rewards/margins": 0.003972449339926243, | |
| "rewards/rejected": -0.09513955563306808, | |
| "sft_loss": 0.9116710424423218, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 2.489442098244054, | |
| "grad_norm": 0.425468772649765, | |
| "learning_rate": 3.4671631608781815e-07, | |
| "logits/chosen": -0.3139536380767822, | |
| "logits/rejected": -0.32965949177742004, | |
| "logps/chosen": -0.9703924059867859, | |
| "logps/rejected": -1.079158067703247, | |
| "loss": 1.0439, | |
| "odds_ratio_loss": 0.7353022694587708, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.09703925251960754, | |
| "rewards/margins": 0.010876556858420372, | |
| "rewards/rejected": -0.10791579633951187, | |
| "sft_loss": 0.9703924059867859, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 2.507223827517226, | |
| "grad_norm": 0.6458228826522827, | |
| "learning_rate": 3.234222580780405e-07, | |
| "logits/chosen": -0.3632466197013855, | |
| "logits/rejected": -0.3340745270252228, | |
| "logps/chosen": -0.942143440246582, | |
| "logps/rejected": -0.9809234738349915, | |
| "loss": 1.0153, | |
| "odds_ratio_loss": 0.7311049103736877, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09421434998512268, | |
| "rewards/margins": 0.0038779997266829014, | |
| "rewards/rejected": -0.09809235483407974, | |
| "sft_loss": 0.942143440246582, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 2.525005556790398, | |
| "grad_norm": 0.7571399211883545, | |
| "learning_rate": 3.0088398414982375e-07, | |
| "logits/chosen": -0.40216293931007385, | |
| "logits/rejected": -0.3554636836051941, | |
| "logps/chosen": -0.9506216049194336, | |
| "logps/rejected": -1.1040918827056885, | |
| "loss": 1.0238, | |
| "odds_ratio_loss": 0.7313109636306763, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.09506215900182724, | |
| "rewards/margins": 0.015347021631896496, | |
| "rewards/rejected": -0.11040918529033661, | |
| "sft_loss": 0.9506216049194336, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 2.54278728606357, | |
| "grad_norm": 0.41928017139434814, | |
| "learning_rate": 2.7910932038184487e-07, | |
| "logits/chosen": -0.38035768270492554, | |
| "logits/rejected": -0.43410953879356384, | |
| "logps/chosen": -0.9504894018173218, | |
| "logps/rejected": -1.033362627029419, | |
| "loss": 1.0219, | |
| "odds_ratio_loss": 0.7138369083404541, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.0950489416718483, | |
| "rewards/margins": 0.008287337608635426, | |
| "rewards/rejected": -0.10333627462387085, | |
| "sft_loss": 0.9504894018173218, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 2.5605690153367417, | |
| "grad_norm": 0.6664097905158997, | |
| "learning_rate": 2.5810582770057325e-07, | |
| "logits/chosen": -0.3502410054206848, | |
| "logits/rejected": -0.31972765922546387, | |
| "logps/chosen": -0.912204384803772, | |
| "logps/rejected": -1.0270380973815918, | |
| "loss": 0.9827, | |
| "odds_ratio_loss": 0.7054314613342285, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09122045338153839, | |
| "rewards/margins": 0.011483349837362766, | |
| "rewards/rejected": -0.10270379483699799, | |
| "sft_loss": 0.912204384803772, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 2.578350744609913, | |
| "grad_norm": 0.5214207768440247, | |
| "learning_rate": 2.3788079925484402e-07, | |
| "logits/chosen": -0.2704157829284668, | |
| "logits/rejected": -0.30042511224746704, | |
| "logps/chosen": -0.980503261089325, | |
| "logps/rejected": -1.0476016998291016, | |
| "loss": 1.054, | |
| "odds_ratio_loss": 0.7349393963813782, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.0980503261089325, | |
| "rewards/margins": 0.006709852255880833, | |
| "rewards/rejected": -0.10476018488407135, | |
| "sft_loss": 0.980503261089325, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 2.596132473883085, | |
| "grad_norm": 0.3559114336967468, | |
| "learning_rate": 2.1844125788342661e-07, | |
| "logits/chosen": -0.3745304048061371, | |
| "logits/rejected": -0.3963877558708191, | |
| "logps/chosen": -0.8978282809257507, | |
| "logps/rejected": -1.1463072299957275, | |
| "loss": 0.966, | |
| "odds_ratio_loss": 0.6815627813339233, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.08978282660245895, | |
| "rewards/margins": 0.02484789676964283, | |
| "rewards/rejected": -0.11463073641061783, | |
| "sft_loss": 0.8978282809257507, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 2.613914203156257, | |
| "grad_norm": 0.4206191599369049, | |
| "learning_rate": 1.9979395367644428e-07, | |
| "logits/chosen": -0.3081280589103699, | |
| "logits/rejected": -0.2860923111438751, | |
| "logps/chosen": -0.8848710060119629, | |
| "logps/rejected": -1.030397653579712, | |
| "loss": 0.9502, | |
| "odds_ratio_loss": 0.6536397337913513, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08848710358142853, | |
| "rewards/margins": 0.014552672393620014, | |
| "rewards/rejected": -0.10303977876901627, | |
| "sft_loss": 0.8848710060119629, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 2.631695932429429, | |
| "grad_norm": 0.6648186445236206, | |
| "learning_rate": 1.81945361631512e-07, | |
| "logits/chosen": -0.3387419283390045, | |
| "logits/rejected": -0.2922862768173218, | |
| "logps/chosen": -0.927925705909729, | |
| "logps/rejected": -0.9954597353935242, | |
| "loss": 1.0003, | |
| "odds_ratio_loss": 0.7234224081039429, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0927925705909729, | |
| "rewards/margins": 0.006753397174179554, | |
| "rewards/rejected": -0.09954597055912018, | |
| "sft_loss": 0.927925705909729, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 2.6494776617026004, | |
| "grad_norm": 0.5596628189086914, | |
| "learning_rate": 1.6490167940538343e-07, | |
| "logits/chosen": -0.3137277066707611, | |
| "logits/rejected": -0.3255840241909027, | |
| "logps/chosen": -0.9538249969482422, | |
| "logps/rejected": -1.0488290786743164, | |
| "loss": 1.0255, | |
| "odds_ratio_loss": 0.7165058851242065, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.09538250416517258, | |
| "rewards/margins": 0.009500409476459026, | |
| "rewards/rejected": -0.10488291084766388, | |
| "sft_loss": 0.9538249969482422, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 2.6672593909757722, | |
| "grad_norm": 0.4116540849208832, | |
| "learning_rate": 1.4866882516191339e-07, | |
| "logits/chosen": -0.31974849104881287, | |
| "logits/rejected": -0.27599194645881653, | |
| "logps/chosen": -0.9288945198059082, | |
| "logps/rejected": -1.0830228328704834, | |
| "loss": 0.9998, | |
| "odds_ratio_loss": 0.7095054984092712, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.0928894504904747, | |
| "rewards/margins": 0.015412822365760803, | |
| "rewards/rejected": -0.1083022803068161, | |
| "sft_loss": 0.9288945198059082, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.6672593909757722, | |
| "eval_logits/chosen": -0.3320940136909485, | |
| "eval_logits/rejected": -0.29884636402130127, | |
| "eval_logps/chosen": -0.9399133324623108, | |
| "eval_logps/rejected": -1.080655574798584, | |
| "eval_loss": 1.0073015689849854, | |
| "eval_odds_ratio_loss": 0.6738813519477844, | |
| "eval_rewards/accuracies": 0.515999972820282, | |
| "eval_rewards/chosen": -0.09399133920669556, | |
| "eval_rewards/margins": 0.01407422125339508, | |
| "eval_rewards/rejected": -0.10806556046009064, | |
| "eval_runtime": 185.9317, | |
| "eval_samples_per_second": 5.378, | |
| "eval_sft_loss": 0.9399133324623108, | |
| "eval_steps_per_second": 2.689, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.685041120248944, | |
| "grad_norm": 0.6644484996795654, | |
| "learning_rate": 1.3325243551706057e-07, | |
| "logits/chosen": -0.3859871029853821, | |
| "logits/rejected": -0.36218634247779846, | |
| "logps/chosen": -0.9241644144058228, | |
| "logps/rejected": -1.1543761491775513, | |
| "loss": 0.9915, | |
| "odds_ratio_loss": 0.6730437874794006, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.0924164205789566, | |
| "rewards/margins": 0.023021187633275986, | |
| "rewards/rejected": -0.11543761193752289, | |
| "sft_loss": 0.9241644144058228, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 2.702822849522116, | |
| "grad_norm": 0.6883984208106995, | |
| "learning_rate": 1.1865786358165737e-07, | |
| "logits/chosen": -0.3818913400173187, | |
| "logits/rejected": -0.27337896823883057, | |
| "logps/chosen": -0.9033206701278687, | |
| "logps/rejected": -1.0108495950698853, | |
| "loss": 0.9727, | |
| "odds_ratio_loss": 0.6942235827445984, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.09033207595348358, | |
| "rewards/margins": 0.010752884671092033, | |
| "rewards/rejected": -0.10108494758605957, | |
| "sft_loss": 0.9033206701278687, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 2.720604578795288, | |
| "grad_norm": 1.4156850576400757, | |
| "learning_rate": 1.0489017710262311e-07, | |
| "logits/chosen": -0.39080482721328735, | |
| "logits/rejected": -0.3747466206550598, | |
| "logps/chosen": -1.0374637842178345, | |
| "logps/rejected": -1.1824612617492676, | |
| "loss": 1.1147, | |
| "odds_ratio_loss": 0.7718855142593384, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.10374637693166733, | |
| "rewards/margins": 0.014499744400382042, | |
| "rewards/rejected": -0.11824611574411392, | |
| "sft_loss": 1.0374637842178345, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 2.73838630806846, | |
| "grad_norm": 0.4921424984931946, | |
| "learning_rate": 9.195415670326446e-08, | |
| "logits/chosen": -0.326080858707428, | |
| "logits/rejected": -0.321908175945282, | |
| "logps/chosen": -0.9485294222831726, | |
| "logps/rejected": -1.082155704498291, | |
| "loss": 1.0195, | |
| "odds_ratio_loss": 0.7096532583236694, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.09485294669866562, | |
| "rewards/margins": 0.013362633064389229, | |
| "rewards/rejected": -0.1082155704498291, | |
| "sft_loss": 0.9485294222831726, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 2.7561680373416317, | |
| "grad_norm": 0.686665415763855, | |
| "learning_rate": 7.985429422327384e-08, | |
| "logits/chosen": -0.35336002707481384, | |
| "logits/rejected": -0.3244116008281708, | |
| "logps/chosen": -0.9436219930648804, | |
| "logps/rejected": -0.975549578666687, | |
| "loss": 1.0188, | |
| "odds_ratio_loss": 0.7518836855888367, | |
| "rewards/accuracies": 0.4124999940395355, | |
| "rewards/chosen": -0.09436219930648804, | |
| "rewards/margins": 0.0031927600502967834, | |
| "rewards/rejected": -0.09755495190620422, | |
| "sft_loss": 0.9436219930648804, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 2.773949766614803, | |
| "grad_norm": 0.30419808626174927, | |
| "learning_rate": 6.859479115900818e-08, | |
| "logits/chosen": -0.31769606471061707, | |
| "logits/rejected": -0.31846362352371216, | |
| "logps/chosen": -0.9142364263534546, | |
| "logps/rejected": -1.0324945449829102, | |
| "loss": 0.9834, | |
| "odds_ratio_loss": 0.6916245222091675, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.0914236530661583, | |
| "rewards/margins": 0.011825799010694027, | |
| "rewards/rejected": -0.1032494530081749, | |
| "sft_loss": 0.9142364263534546, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 2.791731495887975, | |
| "grad_norm": 1.5349509716033936, | |
| "learning_rate": 5.817955720457902e-08, | |
| "logits/chosen": -0.33953648805618286, | |
| "logits/rejected": -0.297925740480423, | |
| "logps/chosen": -0.9395607709884644, | |
| "logps/rejected": -1.0038203001022339, | |
| "loss": 1.0133, | |
| "odds_ratio_loss": 0.7371524572372437, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09395607560873032, | |
| "rewards/margins": 0.006425946019589901, | |
| "rewards/rejected": -0.10038203001022339, | |
| "sft_loss": 0.9395607709884644, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 2.809513225161147, | |
| "grad_norm": 0.36313971877098083, | |
| "learning_rate": 4.861220889427199e-08, | |
| "logits/chosen": -0.35685330629348755, | |
| "logits/rejected": -0.35064131021499634, | |
| "logps/chosen": -0.9390374422073364, | |
| "logps/rejected": -1.019951581954956, | |
| "loss": 1.012, | |
| "odds_ratio_loss": 0.7297292351722717, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09390375763177872, | |
| "rewards/margins": 0.008091414347290993, | |
| "rewards/rejected": -0.10199517011642456, | |
| "sft_loss": 0.9390374422073364, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 2.827294954434319, | |
| "grad_norm": 0.26599186658859253, | |
| "learning_rate": 3.9896068346758074e-08, | |
| "logits/chosen": -0.39413073658943176, | |
| "logits/rejected": -0.38061630725860596, | |
| "logps/chosen": -0.948017954826355, | |
| "logps/rejected": -1.034618616104126, | |
| "loss": 1.0172, | |
| "odds_ratio_loss": 0.6922141313552856, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.09480179846286774, | |
| "rewards/margins": 0.008660053834319115, | |
| "rewards/rejected": -0.1034618467092514, | |
| "sft_loss": 0.948017954826355, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 2.8450766837074903, | |
| "grad_norm": 0.9985164403915405, | |
| "learning_rate": 3.203416211153832e-08, | |
| "logits/chosen": -0.3526967763900757, | |
| "logits/rejected": -0.25582748651504517, | |
| "logps/chosen": -0.9348894357681274, | |
| "logps/rejected": -1.0583240985870361, | |
| "loss": 1.0071, | |
| "odds_ratio_loss": 0.7220235466957092, | |
| "rewards/accuracies": 0.4312500059604645, | |
| "rewards/chosen": -0.09348894655704498, | |
| "rewards/margins": 0.01234346441924572, | |
| "rewards/rejected": -0.10583242028951645, | |
| "sft_loss": 0.9348894357681274, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 2.8628584129806622, | |
| "grad_norm": 0.4895220994949341, | |
| "learning_rate": 2.5029220118019393e-08, | |
| "logits/chosen": -0.3774477243423462, | |
| "logits/rejected": -0.34018778800964355, | |
| "logps/chosen": -0.9445845484733582, | |
| "logps/rejected": -0.9962360262870789, | |
| "loss": 1.0176, | |
| "odds_ratio_loss": 0.7305063009262085, | |
| "rewards/accuracies": 0.4124999940395355, | |
| "rewards/chosen": -0.0944584533572197, | |
| "rewards/margins": 0.0051651508547365665, | |
| "rewards/rejected": -0.09962360560894012, | |
| "sft_loss": 0.9445845484733582, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 2.880640142253834, | |
| "grad_norm": 0.39454635977745056, | |
| "learning_rate": 1.8883674727586122e-08, | |
| "logits/chosen": -0.3457157611846924, | |
| "logits/rejected": -0.33168259263038635, | |
| "logps/chosen": -0.8693550825119019, | |
| "logps/rejected": -1.09225332736969, | |
| "loss": 0.9328, | |
| "odds_ratio_loss": 0.6342187523841858, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.08693551272153854, | |
| "rewards/margins": 0.022289803251624107, | |
| "rewards/rejected": -0.1092253178358078, | |
| "sft_loss": 0.8693550825119019, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 2.898421871527006, | |
| "grad_norm": 0.29763612151145935, | |
| "learning_rate": 1.3599659889000639e-08, | |
| "logits/chosen": -0.26188623905181885, | |
| "logits/rejected": -0.27545788884162903, | |
| "logps/chosen": -0.9086050987243652, | |
| "logps/rejected": -0.9591732025146484, | |
| "loss": 0.9816, | |
| "odds_ratio_loss": 0.7299038171768188, | |
| "rewards/accuracies": 0.4124999940395355, | |
| "rewards/chosen": -0.0908605083823204, | |
| "rewards/margins": 0.005056814290583134, | |
| "rewards/rejected": -0.09591732919216156, | |
| "sft_loss": 0.9086050987243652, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 2.916203600800178, | |
| "grad_norm": 3.087757110595703, | |
| "learning_rate": 9.179010397421528e-09, | |
| "logits/chosen": -0.29684725403785706, | |
| "logits/rejected": -0.26544058322906494, | |
| "logps/chosen": -1.0444749593734741, | |
| "logps/rejected": -1.1464588642120361, | |
| "loss": 1.1156, | |
| "odds_ratio_loss": 0.7117230892181396, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.10444750636816025, | |
| "rewards/margins": 0.010198366828262806, | |
| "rewards/rejected": -0.11464587599039078, | |
| "sft_loss": 1.0444749593734741, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 2.93398533007335, | |
| "grad_norm": 0.7389609813690186, | |
| "learning_rate": 5.623261257296509e-09, | |
| "logits/chosen": -0.33190470933914185, | |
| "logits/rejected": -0.2921023964881897, | |
| "logps/chosen": -0.8605577349662781, | |
| "logps/rejected": -0.9687950015068054, | |
| "loss": 0.9291, | |
| "odds_ratio_loss": 0.6854843497276306, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.08605578541755676, | |
| "rewards/margins": 0.010823719203472137, | |
| "rewards/rejected": -0.0968794971704483, | |
| "sft_loss": 0.8605577349662781, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 2.9517670593465217, | |
| "grad_norm": 0.49204200506210327, | |
| "learning_rate": 2.933647149357122e-09, | |
| "logits/chosen": -0.3684224784374237, | |
| "logits/rejected": -0.3360394537448883, | |
| "logps/chosen": -0.9260095357894897, | |
| "logps/rejected": -1.059597373008728, | |
| "loss": 0.9945, | |
| "odds_ratio_loss": 0.6844674348831177, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.09260095655918121, | |
| "rewards/margins": 0.013358776457607746, | |
| "rewards/rejected": -0.10595973581075668, | |
| "sft_loss": 0.9260095357894897, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 2.969548788619693, | |
| "grad_norm": 0.4070994257926941, | |
| "learning_rate": 1.1111020018930717e-09, | |
| "logits/chosen": -0.2591468393802643, | |
| "logits/rejected": -0.31176748871803284, | |
| "logps/chosen": -0.9283815622329712, | |
| "logps/rejected": -0.9903603792190552, | |
| "loss": 1.0009, | |
| "odds_ratio_loss": 0.7251425981521606, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.09283814579248428, | |
| "rewards/margins": 0.006197893992066383, | |
| "rewards/rejected": -0.09903603792190552, | |
| "sft_loss": 0.9283815622329712, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 2.987330517892865, | |
| "grad_norm": 0.31971636414527893, | |
| "learning_rate": 1.5625866646051813e-10, | |
| "logits/chosen": -0.3598848283290863, | |
| "logits/rejected": -0.3403863310813904, | |
| "logps/chosen": -0.9049466252326965, | |
| "logps/rejected": -1.057483434677124, | |
| "loss": 0.9695, | |
| "odds_ratio_loss": 0.6452642679214478, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.09049466997385025, | |
| "rewards/margins": 0.015253685414791107, | |
| "rewards/rejected": -0.10574835538864136, | |
| "sft_loss": 0.9049466252326965, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 2.997999555456768, | |
| "step": 1686, | |
| "total_flos": 1.8817568285770383e+18, | |
| "train_loss": 1.0353579054523618, | |
| "train_runtime": 16950.0138, | |
| "train_samples_per_second": 1.593, | |
| "train_steps_per_second": 0.099 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1686, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "total_flos": 1.8817568285770383e+18, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |