| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.9748953974895398, | |
| "eval_steps": 500, | |
| "global_step": 118, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.016736401673640166, | |
| "grad_norm": 254.03603103806802, | |
| "learning_rate": 8.333333333333332e-09, | |
| "logits/chosen": 0.40769851207733154, | |
| "logits/rejected": 0.6983045935630798, | |
| "logps/chosen": -597.6331176757812, | |
| "logps/pi_response": -454.7916259765625, | |
| "logps/ref_response": -454.7916259765625, | |
| "logps/rejected": -933.78369140625, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.16736401673640167, | |
| "grad_norm": 223.89448476390103, | |
| "learning_rate": 8.333333333333334e-08, | |
| "logits/chosen": 0.3617916703224182, | |
| "logits/rejected": 0.8798990249633789, | |
| "logps/chosen": -520.3926391601562, | |
| "logps/pi_response": -373.90179443359375, | |
| "logps/ref_response": -369.4568176269531, | |
| "logps/rejected": -942.7015991210938, | |
| "loss": 0.6814, | |
| "rewards/accuracies": 0.5277777910232544, | |
| "rewards/chosen": -0.003062439849600196, | |
| "rewards/margins": 0.031748898327350616, | |
| "rewards/rejected": -0.034811343997716904, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.33472803347280333, | |
| "grad_norm": 275.758299798461, | |
| "learning_rate": 9.860114570402053e-08, | |
| "logits/chosen": 0.41715487837791443, | |
| "logits/rejected": 0.8884197473526001, | |
| "logps/chosen": -551.1149291992188, | |
| "logps/pi_response": -498.0843200683594, | |
| "logps/ref_response": -378.30291748046875, | |
| "logps/rejected": -1005.5443115234375, | |
| "loss": 0.5604, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.09888347238302231, | |
| "rewards/margins": 1.0031945705413818, | |
| "rewards/rejected": -1.1020780801773071, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.502092050209205, | |
| "grad_norm": 151.4646821692468, | |
| "learning_rate": 9.305218058836777e-08, | |
| "logits/chosen": 0.5644534826278687, | |
| "logits/rejected": 0.9862662553787231, | |
| "logps/chosen": -576.3702392578125, | |
| "logps/pi_response": -784.0135498046875, | |
| "logps/ref_response": -373.52685546875, | |
| "logps/rejected": -1288.76220703125, | |
| "loss": 0.4456, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.5222317576408386, | |
| "rewards/margins": 3.5375170707702637, | |
| "rewards/rejected": -4.059748649597168, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.6694560669456067, | |
| "grad_norm": 99.31936965278155, | |
| "learning_rate": 8.374915007591053e-08, | |
| "logits/chosen": 0.6754584312438965, | |
| "logits/rejected": 1.3761770725250244, | |
| "logps/chosen": -637.95263671875, | |
| "logps/pi_response": -662.8466796875, | |
| "logps/ref_response": -352.9124450683594, | |
| "logps/rejected": -1316.7098388671875, | |
| "loss": 0.3955, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.1482101678848267, | |
| "rewards/margins": 2.7251365184783936, | |
| "rewards/rejected": -3.8733463287353516, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.8368200836820083, | |
| "grad_norm": 81.4813373754742, | |
| "learning_rate": 7.150326011382603e-08, | |
| "logits/chosen": 0.8304530382156372, | |
| "logits/rejected": 1.3797433376312256, | |
| "logps/chosen": -703.5577392578125, | |
| "logps/pi_response": -740.5267944335938, | |
| "logps/ref_response": -367.9375305175781, | |
| "logps/rejected": -1414.14453125, | |
| "loss": 0.389, | |
| "rewards/accuracies": 0.8343750238418579, | |
| "rewards/chosen": -1.441198468208313, | |
| "rewards/margins": 3.4418697357177734, | |
| "rewards/rejected": -4.883068084716797, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 1.00418410041841, | |
| "grad_norm": 68.59895920394845, | |
| "learning_rate": 5.738232820012406e-08, | |
| "logits/chosen": 0.8587929606437683, | |
| "logits/rejected": 1.2634632587432861, | |
| "logps/chosen": -588.1586303710938, | |
| "logps/pi_response": -831.5247192382812, | |
| "logps/ref_response": -366.019287109375, | |
| "logps/rejected": -1391.0379638671875, | |
| "loss": 0.3563, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.0079401731491089, | |
| "rewards/margins": 4.322751045227051, | |
| "rewards/rejected": -5.330691337585449, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 1.1715481171548117, | |
| "grad_norm": 78.3617679229796, | |
| "learning_rate": 4.2617671799875946e-08, | |
| "logits/chosen": 0.7778602838516235, | |
| "logits/rejected": 1.3484394550323486, | |
| "logps/chosen": -664.4850463867188, | |
| "logps/pi_response": -791.7720947265625, | |
| "logps/ref_response": -386.8902893066406, | |
| "logps/rejected": -1374.6849365234375, | |
| "loss": 0.3516, | |
| "rewards/accuracies": 0.846875011920929, | |
| "rewards/chosen": -1.1506381034851074, | |
| "rewards/margins": 3.6012942790985107, | |
| "rewards/rejected": -4.751932621002197, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 1.3389121338912133, | |
| "grad_norm": 54.85306379624032, | |
| "learning_rate": 2.8496739886173992e-08, | |
| "logits/chosen": 0.8767589330673218, | |
| "logits/rejected": 1.290276288986206, | |
| "logps/chosen": -640.9560546875, | |
| "logps/pi_response": -780.9495239257812, | |
| "logps/ref_response": -370.62164306640625, | |
| "logps/rejected": -1382.840576171875, | |
| "loss": 0.3358, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -1.0898466110229492, | |
| "rewards/margins": 3.9194438457489014, | |
| "rewards/rejected": -5.0092902183532715, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 1.506276150627615, | |
| "grad_norm": 88.99295644400407, | |
| "learning_rate": 1.6250849924089483e-08, | |
| "logits/chosen": 0.710281491279602, | |
| "logits/rejected": 1.3649709224700928, | |
| "logps/chosen": -608.7789306640625, | |
| "logps/pi_response": -748.4505615234375, | |
| "logps/ref_response": -349.5060729980469, | |
| "logps/rejected": -1372.438232421875, | |
| "loss": 0.3777, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": -1.0407220125198364, | |
| "rewards/margins": 3.8326003551483154, | |
| "rewards/rejected": -4.873322486877441, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 1.6736401673640167, | |
| "grad_norm": 63.93568226735586, | |
| "learning_rate": 6.947819411632222e-09, | |
| "logits/chosen": 0.664978563785553, | |
| "logits/rejected": 1.2786552906036377, | |
| "logps/chosen": -617.9248657226562, | |
| "logps/pi_response": -712.550048828125, | |
| "logps/ref_response": -370.06866455078125, | |
| "logps/rejected": -1386.747314453125, | |
| "loss": 0.3394, | |
| "rewards/accuracies": 0.8687499761581421, | |
| "rewards/chosen": -1.01349937915802, | |
| "rewards/margins": 3.325646162033081, | |
| "rewards/rejected": -4.339145183563232, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 1.8410041841004183, | |
| "grad_norm": 48.91170773014705, | |
| "learning_rate": 1.3988542959794625e-09, | |
| "logits/chosen": 0.6376602649688721, | |
| "logits/rejected": 1.2136269807815552, | |
| "logps/chosen": -611.235595703125, | |
| "logps/pi_response": -798.3106689453125, | |
| "logps/ref_response": -364.35662841796875, | |
| "logps/rejected": -1384.1715087890625, | |
| "loss": 0.3528, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -1.1132670640945435, | |
| "rewards/margins": 3.8225269317626953, | |
| "rewards/rejected": -4.935793876647949, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 1.9748953974895398, | |
| "step": 118, | |
| "total_flos": 0.0, | |
| "train_loss": 0.4142875792616505, | |
| "train_runtime": 5502.1117, | |
| "train_samples_per_second": 5.555, | |
| "train_steps_per_second": 0.021 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 118, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 2, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |