trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 48,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.02,
      "learning_rate": 1e-07,
      "logits/chosen": -2.7669034004211426,
      "logits/rejected": -2.7532925605773926,
      "logps/chosen": -201.60781860351562,
      "logps/pi_response": -160.00347900390625,
      "logps/ref_response": -160.00347900390625,
      "logps/rejected": -279.8238525390625,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.83504027183137e-07,
      "logits/chosen": -2.659675121307373,
      "logits/rejected": -2.595369815826416,
      "logps/chosen": -237.69110107421875,
      "logps/pi_response": -167.47540283203125,
      "logps/ref_response": -166.28582763671875,
      "logps/rejected": -253.0725860595703,
      "loss": 0.6844,
      "rewards/accuracies": 0.5138888955116272,
      "rewards/chosen": -0.027391234412789345,
      "rewards/margins": 0.022421281784772873,
      "rewards/rejected": -0.04981251433491707,
      "step": 10
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.643105808261596e-07,
      "logits/chosen": -2.712937116622925,
      "logits/rejected": -2.677903413772583,
      "logps/chosen": -255.10574340820312,
      "logps/pi_response": -150.52651977539062,
      "logps/ref_response": -159.79461669921875,
      "logps/rejected": -300.8284912109375,
      "loss": 0.6186,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.0877402126789093,
      "rewards/margins": 0.2590124309062958,
      "rewards/rejected": -0.3467526435852051,
      "step": 20
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8676665440207977e-07,
      "logits/chosen": -2.6582727432250977,
      "logits/rejected": -2.5892062187194824,
      "logps/chosen": -297.04840087890625,
      "logps/pi_response": -204.99075317382812,
      "logps/ref_response": -186.3592987060547,
      "logps/rejected": -361.45928955078125,
      "loss": 0.5902,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.3797666132450104,
      "rewards/margins": 0.5780380964279175,
      "rewards/rejected": -0.957804799079895,
      "step": 30
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1500545527530544e-08,
      "logits/chosen": -2.689389705657959,
      "logits/rejected": -2.6197242736816406,
      "logps/chosen": -274.03094482421875,
      "logps/pi_response": -184.63693237304688,
      "logps/ref_response": -167.79286193847656,
      "logps/rejected": -359.7801208496094,
      "loss": 0.5791,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.4090074598789215,
      "rewards/margins": 0.5597021579742432,
      "rewards/rejected": -0.9687095880508423,
      "step": 40
    },
    {
      "epoch": 1.0,
      "step": 48,
      "total_flos": 0.0,
      "train_loss": 0.6088160475095113,
      "train_runtime": 2903.6636,
      "train_samples_per_second": 4.211,
      "train_steps_per_second": 0.017
    }
  ],
  "logging_steps": 10,
  "max_steps": 48,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}