{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9961977186311787, "eval_steps": 100, "global_step": 131, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.01, "grad_norm": 7.4010863350398886, "learning_rate": 3.571428571428571e-08, "logits/chosen": -0.4023560881614685, "logits/rejected": -0.03399945795536041, "logps/chosen": -348.97918701171875, "logps/rejected": -524.31689453125, "loss": 0.6931, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1 }, { "epoch": 0.08, "grad_norm": 7.936227880426271, "learning_rate": 3.5714285714285716e-07, "logits/chosen": -0.5313352942466736, "logits/rejected": -0.32142913341522217, "logps/chosen": -339.4945068359375, "logps/rejected": -743.97900390625, "loss": 0.6925, "rewards/accuracies": 0.4652777910232544, "rewards/chosen": -0.00040899438317865133, "rewards/margins": 0.0011602669255807996, "rewards/rejected": -0.001569261308759451, "step": 10 }, { "epoch": 0.15, "grad_norm": 11.613933346354703, "learning_rate": 4.967625656594781e-07, "logits/chosen": -0.5781797170639038, "logits/rejected": -0.3319936990737915, "logps/chosen": -303.1843566894531, "logps/rejected": -763.7269897460938, "loss": 0.6663, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -0.006486949510872364, "rewards/margins": 0.046987053006887436, "rewards/rejected": -0.053474001586437225, "step": 20 }, { "epoch": 0.23, "grad_norm": 6.944322603367183, "learning_rate": 4.772810749506809e-07, "logits/chosen": -0.4766184687614441, "logits/rejected": -0.320529967546463, "logps/chosen": -367.22760009765625, "logps/rejected": -772.4141845703125, "loss": 0.5753, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -0.0389162078499794, "rewards/margins": 0.31977471709251404, "rewards/rejected": -0.35869091749191284, "step": 30 }, { "epoch": 0.3, "grad_norm": 6.585270206448342, "learning_rate": 4.415111107797445e-07, "logits/chosen": -0.5007311105728149, "logits/rejected": -0.31793397665023804, "logps/chosen": -339.66522216796875, "logps/rejected": -744.0791015625, "loss": 0.4984, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -0.06443960964679718, "rewards/margins": 0.832168459892273, "rewards/rejected": -0.8966080546379089, "step": 40 }, { "epoch": 0.38, "grad_norm": 5.431472038503189, "learning_rate": 3.920161866827889e-07, "logits/chosen": -0.5240360498428345, "logits/rejected": -0.3544757664203644, "logps/chosen": -341.50177001953125, "logps/rejected": -987.6541748046875, "loss": 0.3943, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -0.15628758072853088, "rewards/margins": 2.394836902618408, "rewards/rejected": -2.5511245727539062, "step": 50 }, { "epoch": 0.46, "grad_norm": 5.4415990048052185, "learning_rate": 3.323434381080199e-07, "logits/chosen": -0.5124906301498413, "logits/rejected": -0.3748542368412018, "logps/chosen": -387.4263916015625, "logps/rejected": -991.39697265625, "loss": 0.3611, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -0.2954562306404114, "rewards/margins": 2.682974338531494, "rewards/rejected": -2.97843074798584, "step": 60 }, { "epoch": 0.53, "grad_norm": 4.921715920068345, "learning_rate": 2.667694110977506e-07, "logits/chosen": -0.5369243025779724, "logits/rejected": -0.4064721167087555, "logps/chosen": -352.8229675292969, "logps/rejected": -1028.784912109375, "loss": 0.3188, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": -0.32619190216064453, "rewards/margins": 3.4282054901123047, "rewards/rejected": -3.754396915435791, "step": 70 }, { "epoch": 0.61, "grad_norm": 4.517789071403685, "learning_rate": 1.9999357655598891e-07, "logits/chosen": -0.5413460731506348, "logits/rejected": -0.48942112922668457, "logps/chosen": -381.2135925292969, "logps/rejected": -1212.373779296875, "loss": 0.2756, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -0.3778177797794342, "rewards/margins": 4.809152126312256, "rewards/rejected": -5.186970233917236, "step": 80 }, { "epoch": 0.68, "grad_norm": 4.895205453434471, "learning_rate": 1.3680153490759073e-07, "logits/chosen": -0.4575203061103821, "logits/rejected": -0.43544453382492065, "logps/chosen": -313.4459533691406, "logps/rejected": -1254.440673828125, "loss": 0.276, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -0.2767553925514221, "rewards/margins": 5.2881364822387695, "rewards/rejected": -5.564891815185547, "step": 90 }, { "epoch": 0.76, "grad_norm": 4.400965922122272, "learning_rate": 8.17220481431074e-08, "logits/chosen": -0.4834328591823578, "logits/rejected": -0.4576188027858734, "logps/chosen": -366.92529296875, "logps/rejected": -1311.052734375, "loss": 0.2708, "rewards/accuracies": 0.875, "rewards/chosen": -0.38915878534317017, "rewards/margins": 5.743791103363037, "rewards/rejected": -6.1329498291015625, "step": 100 }, { "epoch": 0.76, "eval_logits/chosen": -0.4292154312133789, "eval_logits/rejected": -0.47080573439598083, "eval_logps/chosen": -476.12548828125, "eval_logps/rejected": -635.9033203125, "eval_loss": 0.4682200849056244, "eval_rewards/accuracies": 0.925000011920929, "eval_rewards/chosen": -0.1608993411064148, "eval_rewards/margins": 0.9414912462234497, "eval_rewards/rejected": -1.1023906469345093, "eval_runtime": 39.9857, "eval_samples_per_second": 8.003, "eval_steps_per_second": 0.25, "step": 100 }, { "epoch": 0.84, "grad_norm": 4.121824084175862, "learning_rate": 3.8702478614051345e-08, "logits/chosen": -0.3803841173648834, "logits/rejected": -0.4126942753791809, "logps/chosen": -375.005126953125, "logps/rejected": -1269.2353515625, "loss": 0.2576, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -0.4322739243507385, "rewards/margins": 5.517593860626221, "rewards/rejected": -5.949868202209473, "step": 110 }, { "epoch": 0.91, "grad_norm": 3.4792807330473705, "learning_rate": 1.082589479314372e-08, "logits/chosen": -0.4776890277862549, "logits/rejected": -0.5241965055465698, "logps/chosen": -376.96905517578125, "logps/rejected": -1401.2740478515625, "loss": 0.2385, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -0.4306109845638275, "rewards/margins": 5.790417194366455, "rewards/rejected": -6.2210283279418945, "step": 120 }, { "epoch": 0.99, "grad_norm": 5.31082634895388, "learning_rate": 9.011808227865625e-11, "logits/chosen": -0.47645965218544006, "logits/rejected": -0.45030665397644043, "logps/chosen": -378.35400390625, "logps/rejected": -1342.3602294921875, "loss": 0.2593, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -0.346562922000885, "rewards/margins": 5.536257743835449, "rewards/rejected": -5.882820129394531, "step": 130 }, { "epoch": 1.0, "step": 131, "total_flos": 0.0, "train_loss": 0.05898992068894947, "train_runtime": 471.0672, "train_samples_per_second": 17.853, "train_steps_per_second": 0.278 } ], "logging_steps": 10, "max_steps": 131, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 100, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }