| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9984, | |
| "eval_steps": 100, | |
| "global_step": 312, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0032, | |
| "grad_norm": 647945.4912541932, | |
| "learning_rate": 1.5625e-08, | |
| "logits/chosen": -0.34773391485214233, | |
| "logits/rejected": -0.6075438261032104, | |
| "logps/chosen": -72.6761474609375, | |
| "logps/rejected": -90.11207580566406, | |
| "loss": 128855.9062, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.032, | |
| "grad_norm": 973324.1712020065, | |
| "learning_rate": 1.5624999999999999e-07, | |
| "logits/chosen": -0.5611530542373657, | |
| "logits/rejected": -0.5887401103973389, | |
| "logps/chosen": -80.2381591796875, | |
| "logps/rejected": -83.50374603271484, | |
| "loss": 124005.5694, | |
| "rewards/accuracies": 0.4166666567325592, | |
| "rewards/chosen": -0.0019423539051786065, | |
| "rewards/margins": 5.1506802265066653e-05, | |
| "rewards/rejected": -0.0019938608165830374, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.064, | |
| "grad_norm": 619327.407060219, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logits/chosen": -0.6772833466529846, | |
| "logits/rejected": -0.6759974360466003, | |
| "logps/chosen": -103.69559478759766, | |
| "logps/rejected": -107.43603515625, | |
| "loss": 124210.2125, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.004284867085516453, | |
| "rewards/margins": -3.467009082669392e-05, | |
| "rewards/rejected": -0.004250196740031242, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.096, | |
| "grad_norm": 698173.4505162692, | |
| "learning_rate": 4.6874999999999996e-07, | |
| "logits/chosen": -0.7464536428451538, | |
| "logits/rejected": -0.7253994345664978, | |
| "logps/chosen": -90.76727294921875, | |
| "logps/rejected": -93.79044342041016, | |
| "loss": 126548.2375, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.003106231102719903, | |
| "rewards/margins": -0.0005979427369311452, | |
| "rewards/rejected": -0.0025082884822040796, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.128, | |
| "grad_norm": 637174.9970357245, | |
| "learning_rate": 4.857142857142857e-07, | |
| "logits/chosen": -0.7085025906562805, | |
| "logits/rejected": -0.7023540139198303, | |
| "logps/chosen": -87.2509765625, | |
| "logps/rejected": -88.0642318725586, | |
| "loss": 124747.6875, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -0.007863897830247879, | |
| "rewards/margins": -0.0013397409347817302, | |
| "rewards/rejected": -0.006524157710373402, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 759040.4009588562, | |
| "learning_rate": 4.6785714285714283e-07, | |
| "logits/chosen": -0.5708094835281372, | |
| "logits/rejected": -0.55577552318573, | |
| "logps/chosen": -99.05384826660156, | |
| "logps/rejected": -96.9248046875, | |
| "loss": 127056.3875, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.011573193594813347, | |
| "rewards/margins": -0.0007376443827524781, | |
| "rewards/rejected": -0.010835548862814903, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.192, | |
| "grad_norm": 818448.4874125579, | |
| "learning_rate": 4.5e-07, | |
| "logits/chosen": -0.5234788060188293, | |
| "logits/rejected": -0.5684272646903992, | |
| "logps/chosen": -84.0132064819336, | |
| "logps/rejected": -89.70082092285156, | |
| "loss": 124101.0125, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.011890527792274952, | |
| "rewards/margins": 0.0017182690789923072, | |
| "rewards/rejected": -0.013608796522021294, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.224, | |
| "grad_norm": 764315.259548912, | |
| "learning_rate": 4.3214285714285713e-07, | |
| "logits/chosen": -0.672571063041687, | |
| "logits/rejected": -0.6554594039916992, | |
| "logps/chosen": -102.6801986694336, | |
| "logps/rejected": -114.0815658569336, | |
| "loss": 125767.8, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.013903990387916565, | |
| "rewards/margins": 0.0018995633581653237, | |
| "rewards/rejected": -0.01580355316400528, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.256, | |
| "grad_norm": 792832.7721251897, | |
| "learning_rate": 4.142857142857143e-07, | |
| "logits/chosen": -0.6233155131340027, | |
| "logits/rejected": -0.6050644516944885, | |
| "logps/chosen": -89.83741760253906, | |
| "logps/rejected": -96.45980072021484, | |
| "loss": 126646.1, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": -0.011440077796578407, | |
| "rewards/margins": -0.0004714619426522404, | |
| "rewards/rejected": -0.010968615300953388, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.288, | |
| "grad_norm": 810791.4710150602, | |
| "learning_rate": 3.9642857142857137e-07, | |
| "logits/chosen": -0.5288355946540833, | |
| "logits/rejected": -0.507430911064148, | |
| "logps/chosen": -77.9104232788086, | |
| "logps/rejected": -74.20404052734375, | |
| "loss": 126600.7625, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.005952201783657074, | |
| "rewards/margins": -0.001560600707307458, | |
| "rewards/rejected": -0.004391600843518972, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 612814.6572972395, | |
| "learning_rate": 3.785714285714285e-07, | |
| "logits/chosen": -0.6446259617805481, | |
| "logits/rejected": -0.6776315569877625, | |
| "logps/chosen": -92.22976684570312, | |
| "logps/rejected": -100.54733276367188, | |
| "loss": 124326.1, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.011474112048745155, | |
| "rewards/margins": 0.002196565503254533, | |
| "rewards/rejected": -0.013670678250491619, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.352, | |
| "grad_norm": 769940.7880329042, | |
| "learning_rate": 3.607142857142857e-07, | |
| "logits/chosen": -0.5441879630088806, | |
| "logits/rejected": -0.5395065546035767, | |
| "logps/chosen": -64.47439575195312, | |
| "logps/rejected": -78.48651123046875, | |
| "loss": 127264.1375, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.008155420422554016, | |
| "rewards/margins": 0.005157289560884237, | |
| "rewards/rejected": -0.013312709517776966, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.384, | |
| "grad_norm": 781127.2959197527, | |
| "learning_rate": 3.4285714285714286e-07, | |
| "logits/chosen": -0.7074313759803772, | |
| "logits/rejected": -0.6893147230148315, | |
| "logps/chosen": -99.30326843261719, | |
| "logps/rejected": -100.26654815673828, | |
| "loss": 126373.0, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.006027103401720524, | |
| "rewards/margins": -0.0006245746044442058, | |
| "rewards/rejected": -0.005402528680860996, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.416, | |
| "grad_norm": 942915.0070681617, | |
| "learning_rate": 3.25e-07, | |
| "logits/chosen": -0.5311844348907471, | |
| "logits/rejected": -0.5678432583808899, | |
| "logps/chosen": -89.84095001220703, | |
| "logps/rejected": -95.73307800292969, | |
| "loss": 126546.9625, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.005261361598968506, | |
| "rewards/margins": -0.00025905706570483744, | |
| "rewards/rejected": -0.00500230398029089, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.448, | |
| "grad_norm": 802161.2678528542, | |
| "learning_rate": 3.0714285714285716e-07, | |
| "logits/chosen": -0.6184743642807007, | |
| "logits/rejected": -0.6451131701469421, | |
| "logps/chosen": -109.21659088134766, | |
| "logps/rejected": -114.1061019897461, | |
| "loss": 125730.125, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.011759540066123009, | |
| "rewards/margins": 0.0014495229115709662, | |
| "rewards/rejected": -0.01320906262844801, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 866428.7327389624, | |
| "learning_rate": 2.892857142857143e-07, | |
| "logits/chosen": -0.6030551195144653, | |
| "logits/rejected": -0.5557407140731812, | |
| "logps/chosen": -82.86506652832031, | |
| "logps/rejected": -85.31071472167969, | |
| "loss": 125425.025, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.010830635204911232, | |
| "rewards/margins": -6.357554957503453e-05, | |
| "rewards/rejected": -0.010767060332000256, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.512, | |
| "grad_norm": 743330.5276750317, | |
| "learning_rate": 2.714285714285714e-07, | |
| "logits/chosen": -0.5015612840652466, | |
| "logits/rejected": -0.5147450566291809, | |
| "logps/chosen": -82.76224517822266, | |
| "logps/rejected": -91.91256713867188, | |
| "loss": 124215.3, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.010083668865263462, | |
| "rewards/margins": 0.0024900883436203003, | |
| "rewards/rejected": -0.012573758140206337, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.544, | |
| "grad_norm": 863614.5495224567, | |
| "learning_rate": 2.5357142857142855e-07, | |
| "logits/chosen": -0.5797610878944397, | |
| "logits/rejected": -0.5199266672134399, | |
| "logps/chosen": -94.99356842041016, | |
| "logps/rejected": -96.22293090820312, | |
| "loss": 127004.7, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.012711484916508198, | |
| "rewards/margins": 0.004797719419002533, | |
| "rewards/rejected": -0.017509203404188156, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.576, | |
| "grad_norm": 831681.0077569862, | |
| "learning_rate": 2.357142857142857e-07, | |
| "logits/chosen": -0.6032494902610779, | |
| "logits/rejected": -0.579995334148407, | |
| "logps/chosen": -104.5300521850586, | |
| "logps/rejected": -108.78277587890625, | |
| "loss": 125979.4375, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.009229556657373905, | |
| "rewards/margins": 0.004828121047466993, | |
| "rewards/rejected": -0.014057678170502186, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.608, | |
| "grad_norm": 780274.1467706825, | |
| "learning_rate": 2.1785714285714284e-07, | |
| "logits/chosen": -0.7121313810348511, | |
| "logits/rejected": -0.667202353477478, | |
| "logps/chosen": -115.69401550292969, | |
| "logps/rejected": -110.82621765136719, | |
| "loss": 124809.7, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.012815780937671661, | |
| "rewards/margins": -0.0001598205417394638, | |
| "rewards/rejected": -0.012655961327254772, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 774598.0171325745, | |
| "learning_rate": 2e-07, | |
| "logits/chosen": -0.612346351146698, | |
| "logits/rejected": -0.6116153001785278, | |
| "logps/chosen": -91.24519348144531, | |
| "logps/rejected": -97.00153350830078, | |
| "loss": 123650.5375, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.01422748900949955, | |
| "rewards/margins": 0.0018453721422702074, | |
| "rewards/rejected": -0.01607285998761654, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.672, | |
| "grad_norm": 1137683.0365726806, | |
| "learning_rate": 1.8214285714285714e-07, | |
| "logits/chosen": -0.6241598725318909, | |
| "logits/rejected": -0.6161590814590454, | |
| "logps/chosen": -82.91732788085938, | |
| "logps/rejected": -92.75973510742188, | |
| "loss": 125116.0125, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.01631699874997139, | |
| "rewards/margins": 0.002573491772636771, | |
| "rewards/rejected": -0.018890492618083954, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.704, | |
| "grad_norm": 921161.3498685773, | |
| "learning_rate": 1.6428571428571429e-07, | |
| "logits/chosen": -0.6814984083175659, | |
| "logits/rejected": -0.6642488241195679, | |
| "logps/chosen": -134.07284545898438, | |
| "logps/rejected": -134.7923126220703, | |
| "loss": 125720.675, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.01326170563697815, | |
| "rewards/margins": 0.0025010218378156424, | |
| "rewards/rejected": -0.015762727707624435, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.736, | |
| "grad_norm": 813896.4945325998, | |
| "learning_rate": 1.4642857142857143e-07, | |
| "logits/chosen": -0.5411783456802368, | |
| "logits/rejected": -0.5778718590736389, | |
| "logps/chosen": -104.65946197509766, | |
| "logps/rejected": -107.73319244384766, | |
| "loss": 125973.8125, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.012594607658684254, | |
| "rewards/margins": 0.0011098148534074426, | |
| "rewards/rejected": -0.013704421930015087, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.768, | |
| "grad_norm": 1031122.2282012746, | |
| "learning_rate": 1.2857142857142855e-07, | |
| "logits/chosen": -0.6678429841995239, | |
| "logits/rejected": -0.6291283369064331, | |
| "logps/chosen": -104.91682434082031, | |
| "logps/rejected": -111.02679443359375, | |
| "loss": 126001.475, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.008200669661164284, | |
| "rewards/margins": 0.0015530238160863519, | |
| "rewards/rejected": -0.009753693826496601, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 858633.8039080129, | |
| "learning_rate": 1.107142857142857e-07, | |
| "logits/chosen": -0.6295119524002075, | |
| "logits/rejected": -0.6167672872543335, | |
| "logps/chosen": -123.36985778808594, | |
| "logps/rejected": -133.19418334960938, | |
| "loss": 126223.65, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.011947548016905785, | |
| "rewards/margins": 0.006852240767329931, | |
| "rewards/rejected": -0.018799791112542152, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.832, | |
| "grad_norm": 951847.1640935472, | |
| "learning_rate": 9.285714285714286e-08, | |
| "logits/chosen": -0.6834455728530884, | |
| "logits/rejected": -0.7226243615150452, | |
| "logps/chosen": -86.39234924316406, | |
| "logps/rejected": -95.36772155761719, | |
| "loss": 124640.2, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.01843985728919506, | |
| "rewards/margins": 0.003491448936983943, | |
| "rewards/rejected": -0.021931307390332222, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.864, | |
| "grad_norm": 816825.5268517752, | |
| "learning_rate": 7.5e-08, | |
| "logits/chosen": -0.6084921956062317, | |
| "logits/rejected": -0.606655478477478, | |
| "logps/chosen": -95.06122589111328, | |
| "logps/rejected": -100.9395523071289, | |
| "loss": 126797.975, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.013821562752127647, | |
| "rewards/margins": 0.0025993138551712036, | |
| "rewards/rejected": -0.01642087660729885, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.896, | |
| "grad_norm": 823903.2164322428, | |
| "learning_rate": 5.714285714285714e-08, | |
| "logits/chosen": -0.7316595315933228, | |
| "logits/rejected": -0.7817249298095703, | |
| "logps/chosen": -97.38008880615234, | |
| "logps/rejected": -122.05289459228516, | |
| "loss": 122803.6375, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.012123498134315014, | |
| "rewards/margins": 0.006116434000432491, | |
| "rewards/rejected": -0.018239933997392654, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.928, | |
| "grad_norm": 1213103.129361221, | |
| "learning_rate": 3.9285714285714285e-08, | |
| "logits/chosen": -0.7132126092910767, | |
| "logits/rejected": -0.7211403846740723, | |
| "logps/chosen": -115.4140853881836, | |
| "logps/rejected": -124.9251480102539, | |
| "loss": 125220.8875, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.008542357943952084, | |
| "rewards/margins": 0.007235427852720022, | |
| "rewards/rejected": -0.01577778533101082, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 826125.8509083999, | |
| "learning_rate": 2.142857142857143e-08, | |
| "logits/chosen": -0.4794866144657135, | |
| "logits/rejected": -0.48627161979675293, | |
| "logps/chosen": -106.44710540771484, | |
| "logps/rejected": -113.4127197265625, | |
| "loss": 124190.425, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.016785580664873123, | |
| "rewards/margins": 0.002466305159032345, | |
| "rewards/rejected": -0.019251886755228043, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.992, | |
| "grad_norm": 853168.6471782625, | |
| "learning_rate": 3.571428571428571e-09, | |
| "logits/chosen": -0.6391203999519348, | |
| "logits/rejected": -0.6226745843887329, | |
| "logps/chosen": -105.24736022949219, | |
| "logps/rejected": -109.426025390625, | |
| "loss": 122976.65, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.01043027639389038, | |
| "rewards/margins": 0.003538835793733597, | |
| "rewards/rejected": -0.013969110324978828, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.9984, | |
| "step": 312, | |
| "total_flos": 0.0, | |
| "train_loss": 125356.69771634616, | |
| "train_runtime": 2759.785, | |
| "train_samples_per_second": 7.245, | |
| "train_steps_per_second": 0.113 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 312, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |