| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9968652037617555, | |
| "eval_steps": 500, | |
| "global_step": 159, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 3.125e-08, | |
| "logits/chosen": -2.072941541671753, | |
| "logits/rejected": -2.0026817321777344, | |
| "logps/chosen": -474.7008361816406, | |
| "logps/pi_response": -295.3243408203125, | |
| "logps/ref_response": -295.3243408203125, | |
| "logps/rejected": -399.129638671875, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logits/chosen": -2.0674960613250732, | |
| "logits/rejected": -2.0091359615325928, | |
| "logps/chosen": -293.94415283203125, | |
| "logps/pi_response": -184.32736206054688, | |
| "logps/ref_response": -184.39166259765625, | |
| "logps/rejected": -331.1109924316406, | |
| "loss": 0.6873, | |
| "rewards/accuracies": 0.5069444179534912, | |
| "rewards/chosen": -0.005304105579853058, | |
| "rewards/margins": 0.014647711999714375, | |
| "rewards/rejected": -0.019951816648244858, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 4.990353313429303e-07, | |
| "logits/chosen": -1.976543664932251, | |
| "logits/rejected": -1.9456901550292969, | |
| "logps/chosen": -287.80316162109375, | |
| "logps/pi_response": -200.89390563964844, | |
| "logps/ref_response": -194.6990509033203, | |
| "logps/rejected": -415.5035095214844, | |
| "loss": 0.6721, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.28011927008628845, | |
| "rewards/margins": 0.24019071459770203, | |
| "rewards/rejected": -0.5203099846839905, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 4.882681251368548e-07, | |
| "logits/chosen": -1.948282241821289, | |
| "logits/rejected": -1.9048038721084595, | |
| "logps/chosen": -322.1227111816406, | |
| "logps/pi_response": -202.88558959960938, | |
| "logps/ref_response": -198.0297088623047, | |
| "logps/rejected": -393.98382568359375, | |
| "loss": 0.6566, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.31841763854026794, | |
| "rewards/margins": 0.2520221173763275, | |
| "rewards/rejected": -0.5704396963119507, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 4.6604720940421207e-07, | |
| "logits/chosen": -1.99917471408844, | |
| "logits/rejected": -1.9424034357070923, | |
| "logps/chosen": -306.33917236328125, | |
| "logps/pi_response": -221.41934204101562, | |
| "logps/ref_response": -196.04620361328125, | |
| "logps/rejected": -473.10430908203125, | |
| "loss": 0.6508, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.4164486825466156, | |
| "rewards/margins": 0.34302154183387756, | |
| "rewards/rejected": -0.7594702243804932, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 4.3344075855595097e-07, | |
| "logits/chosen": -2.022153615951538, | |
| "logits/rejected": -1.8780553340911865, | |
| "logps/chosen": -300.8150634765625, | |
| "logps/pi_response": -213.9830322265625, | |
| "logps/ref_response": -212.6190185546875, | |
| "logps/rejected": -455.1544494628906, | |
| "loss": 0.6199, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.07085980474948883, | |
| "rewards/margins": 0.4123227000236511, | |
| "rewards/rejected": -0.48318248987197876, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 3.920161866827889e-07, | |
| "logits/chosen": -1.623453140258789, | |
| "logits/rejected": -1.5108482837677002, | |
| "logps/chosen": -327.77496337890625, | |
| "logps/pi_response": -236.1122589111328, | |
| "logps/ref_response": -207.8074493408203, | |
| "logps/rejected": -478.13677978515625, | |
| "loss": 0.6015, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.4635746479034424, | |
| "rewards/margins": 0.47717157006263733, | |
| "rewards/rejected": -0.9407461285591125, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 3.4376480090239047e-07, | |
| "logits/chosen": -1.3422645330429077, | |
| "logits/rejected": -1.035585641860962, | |
| "logps/chosen": -311.18182373046875, | |
| "logps/pi_response": -238.19351196289062, | |
| "logps/ref_response": -195.33450317382812, | |
| "logps/rejected": -481.33416748046875, | |
| "loss": 0.5778, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.45635801553726196, | |
| "rewards/margins": 0.5854658484458923, | |
| "rewards/rejected": -1.0418239831924438, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 2.910060778827554e-07, | |
| "logits/chosen": -1.1403189897537231, | |
| "logits/rejected": -0.7611511945724487, | |
| "logps/chosen": -358.22308349609375, | |
| "logps/pi_response": -249.77072143554688, | |
| "logps/ref_response": -202.05001831054688, | |
| "logps/rejected": -496.70916748046875, | |
| "loss": 0.5643, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.564284086227417, | |
| "rewards/margins": 0.598731517791748, | |
| "rewards/rejected": -1.163015604019165, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 2.3627616503391812e-07, | |
| "logits/chosen": -0.8958919644355774, | |
| "logits/rejected": -0.4801406264305115, | |
| "logps/chosen": -365.9770202636719, | |
| "logps/pi_response": -254.03115844726562, | |
| "logps/ref_response": -202.2326202392578, | |
| "logps/rejected": -469.5523986816406, | |
| "loss": 0.582, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.6682159900665283, | |
| "rewards/margins": 0.44698458909988403, | |
| "rewards/rejected": -1.1152006387710571, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 1.8220596619089573e-07, | |
| "logits/chosen": -0.8831208944320679, | |
| "logits/rejected": -0.5018015503883362, | |
| "logps/chosen": -358.91448974609375, | |
| "logps/pi_response": -286.5412292480469, | |
| "logps/ref_response": -215.90811157226562, | |
| "logps/rejected": -526.8685302734375, | |
| "loss": 0.5697, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -0.6926028728485107, | |
| "rewards/margins": 0.6388980150222778, | |
| "rewards/rejected": -1.331500768661499, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 1.3139467229135998e-07, | |
| "logits/chosen": -0.8882027864456177, | |
| "logits/rejected": -0.3719862103462219, | |
| "logps/chosen": -382.29156494140625, | |
| "logps/pi_response": -279.39959716796875, | |
| "logps/ref_response": -194.23623657226562, | |
| "logps/rejected": -500.37554931640625, | |
| "loss": 0.5822, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -0.8814951181411743, | |
| "rewards/margins": 0.4973115026950836, | |
| "rewards/rejected": -1.3788065910339355, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 8.628481651367875e-08, | |
| "logits/chosen": -0.7373358011245728, | |
| "logits/rejected": -0.3627234399318695, | |
| "logps/chosen": -384.5245666503906, | |
| "logps/pi_response": -285.951171875, | |
| "logps/ref_response": -195.06085205078125, | |
| "logps/rejected": -552.2709350585938, | |
| "loss": 0.5429, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -1.024389624595642, | |
| "rewards/margins": 0.5978730916976929, | |
| "rewards/rejected": -1.6222625970840454, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 4.904486005914027e-08, | |
| "logits/chosen": -0.8074722290039062, | |
| "logits/rejected": -0.4555937349796295, | |
| "logps/chosen": -363.5116271972656, | |
| "logps/pi_response": -276.3110046386719, | |
| "logps/ref_response": -198.39134216308594, | |
| "logps/rejected": -539.1277465820312, | |
| "loss": 0.5798, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.7972265481948853, | |
| "rewards/margins": 0.6627088785171509, | |
| "rewards/rejected": -1.4599354267120361, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 2.1464952759020856e-08, | |
| "logits/chosen": -0.7802283763885498, | |
| "logits/rejected": -0.35213881731033325, | |
| "logps/chosen": -357.5544128417969, | |
| "logps/pi_response": -271.56719970703125, | |
| "logps/ref_response": -197.29266357421875, | |
| "logps/rejected": -542.0772094726562, | |
| "loss": 0.5593, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.7836874723434448, | |
| "rewards/margins": 0.6558796167373657, | |
| "rewards/rejected": -1.4395670890808105, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 4.8708793644441086e-09, | |
| "logits/chosen": -0.9855061769485474, | |
| "logits/rejected": -0.5764757990837097, | |
| "logps/chosen": -366.1041564941406, | |
| "logps/pi_response": -283.294921875, | |
| "logps/ref_response": -215.0994873046875, | |
| "logps/rejected": -516.2839965820312, | |
| "loss": 0.5699, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.7308257818222046, | |
| "rewards/margins": 0.46794968843460083, | |
| "rewards/rejected": -1.1987755298614502, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 159, | |
| "total_flos": 0.0, | |
| "train_loss": 0.5937920936248587, | |
| "train_runtime": 4171.3796, | |
| "train_samples_per_second": 4.885, | |
| "train_steps_per_second": 0.038 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 159, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |