{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9874476987447699, "eval_steps": 500, "global_step": 59, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.016736401673640166, "grad_norm": 126.2238527844094, "learning_rate": 1.6666666666666664e-08, "logits/chosen": -0.8470035195350647, "logits/rejected": -0.5631465315818787, "logps/chosen": -390.660400390625, "logps/pi_response": -197.14308166503906, "logps/ref_response": -197.14308166503906, "logps/rejected": -774.7786865234375, "loss": 0.6931, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1 }, { "epoch": 0.16736401673640167, "grad_norm": 136.14999220764295, "learning_rate": 9.860114570402053e-08, "logits/chosen": -0.74688321352005, "logits/rejected": -0.4899700880050659, "logps/chosen": -478.9698486328125, "logps/pi_response": -176.40945434570312, "logps/ref_response": -174.20945739746094, "logps/rejected": -738.7280883789062, "loss": 0.6789, "rewards/accuracies": 0.5520833134651184, "rewards/chosen": -0.008676827885210514, "rewards/margins": 0.021465951576828957, "rewards/rejected": -0.030142780393362045, "step": 10 }, { "epoch": 0.33472803347280333, "grad_norm": 112.25884202942129, "learning_rate": 8.374915007591053e-08, "logits/chosen": -0.6827807426452637, "logits/rejected": -0.48651519417762756, "logps/chosen": -466.18634033203125, "logps/pi_response": -313.5439758300781, "logps/ref_response": -195.1559295654297, "logps/rejected": -830.7267456054688, "loss": 0.5474, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -0.20106668770313263, "rewards/margins": 0.9386097192764282, "rewards/rejected": -1.1396763324737549, "step": 20 }, { "epoch": 0.502092050209205, "grad_norm": 71.37245318602794, "learning_rate": 5.738232820012406e-08, "logits/chosen": -0.5754955410957336, "logits/rejected": -0.37761348485946655, "logps/chosen": -527.6432495117188, "logps/pi_response": -468.9288024902344, "logps/ref_response": -188.6576385498047, "logps/rejected": -963.0016479492188, "loss": 0.4406, "rewards/accuracies": 0.846875011920929, "rewards/chosen": -0.714775562286377, "rewards/margins": 2.008810520172119, "rewards/rejected": -2.723586320877075, "step": 30 }, { "epoch": 0.6694560669456067, "grad_norm": 91.06844156551222, "learning_rate": 2.8496739886173992e-08, "logits/chosen": -0.4327356219291687, "logits/rejected": -0.13516364991664886, "logps/chosen": -603.9708251953125, "logps/pi_response": -443.88800048828125, "logps/ref_response": -201.4982147216797, "logps/rejected": -1077.4971923828125, "loss": 0.4142, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -1.268905520439148, "rewards/margins": 2.0063252449035645, "rewards/rejected": -3.275231122970581, "step": 40 }, { "epoch": 0.8368200836820083, "grad_norm": 102.29535857316833, "learning_rate": 6.947819411632222e-09, "logits/chosen": -0.4187712073326111, "logits/rejected": -0.18879997730255127, "logps/chosen": -592.0279541015625, "logps/pi_response": -473.74993896484375, "logps/ref_response": -189.38746643066406, "logps/rejected": -1088.6204833984375, "loss": 0.411, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -1.303727388381958, "rewards/margins": 2.3152670860290527, "rewards/rejected": -3.6189944744110107, "step": 50 }, { "epoch": 0.9874476987447699, "step": 59, "total_flos": 0.0, "train_loss": 0.49046951229289426, "train_runtime": 2660.7207, "train_samples_per_second": 5.744, "train_steps_per_second": 0.022 } ], "logging_steps": 10, "max_steps": 59, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 100, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }