trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9874476987447699,
  "eval_steps": 500,
  "global_step": 59,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.016736401673640166,
      "grad_norm": 11.574011045378546,
      "learning_rate": 8.333333333333333e-08,
      "logits/chosen": -2.820087432861328,
      "logits/rejected": -2.752105474472046,
      "logps/chosen": -202.43484497070312,
      "logps/pi_response": -118.15838623046875,
      "logps/ref_response": -118.15838623046875,
      "logps/rejected": -230.86993408203125,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.16736401673640167,
      "grad_norm": 12.77979453970249,
      "learning_rate": 4.930057285201027e-07,
      "logits/chosen": -2.7481529712677,
      "logits/rejected": -2.7059879302978516,
      "logps/chosen": -218.31971740722656,
      "logps/pi_response": -117.09883880615234,
      "logps/ref_response": -116.02430725097656,
      "logps/rejected": -268.5631103515625,
      "loss": 0.6858,
      "rewards/accuracies": 0.5486111044883728,
      "rewards/chosen": -0.04048130288720131,
      "rewards/margins": 0.021006720140576363,
      "rewards/rejected": -0.061488017439842224,
      "step": 10
    },
    {
      "epoch": 0.33472803347280333,
      "grad_norm": 26.782381392165316,
      "learning_rate": 4.187457503795526e-07,
      "logits/chosen": -2.763244152069092,
      "logits/rejected": -2.7215287685394287,
      "logps/chosen": -253.488037109375,
      "logps/pi_response": -115.9146499633789,
      "logps/ref_response": -116.65571594238281,
      "logps/rejected": -306.472900390625,
      "loss": 0.6316,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.23194393515586853,
      "rewards/margins": 0.27720019221305847,
      "rewards/rejected": -0.509144127368927,
      "step": 20
    },
    {
      "epoch": 0.502092050209205,
      "grad_norm": 15.90326389389994,
      "learning_rate": 2.8691164100062034e-07,
      "logits/chosen": -2.7404398918151855,
      "logits/rejected": -2.6926448345184326,
      "logps/chosen": -287.66229248046875,
      "logps/pi_response": -152.1663055419922,
      "logps/ref_response": -120.6534652709961,
      "logps/rejected": -356.522216796875,
      "loss": 0.5848,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.5879844427108765,
      "rewards/margins": 0.4546080529689789,
      "rewards/rejected": -1.0425926446914673,
      "step": 30
    },
    {
      "epoch": 0.6694560669456067,
      "grad_norm": 19.24531616262288,
      "learning_rate": 1.4248369943086995e-07,
      "logits/chosen": -2.6505520343780518,
      "logits/rejected": -2.614680051803589,
      "logps/chosen": -302.36932373046875,
      "logps/pi_response": -170.94439697265625,
      "logps/ref_response": -112.9241943359375,
      "logps/rejected": -389.4826965332031,
      "loss": 0.5442,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7899842858314514,
      "rewards/margins": 0.5402624607086182,
      "rewards/rejected": -1.3302466869354248,
      "step": 40
    },
    {
      "epoch": 0.8368200836820083,
      "grad_norm": 21.405822400938703,
      "learning_rate": 3.473909705816111e-08,
      "logits/chosen": -2.6072592735290527,
      "logits/rejected": -2.5605452060699463,
      "logps/chosen": -314.13116455078125,
      "logps/pi_response": -197.56021118164062,
      "logps/ref_response": -112.64766693115234,
      "logps/rejected": -399.7181701660156,
      "loss": 0.5407,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -1.0041061639785767,
      "rewards/margins": 0.6152234673500061,
      "rewards/rejected": -1.6193294525146484,
      "step": 50
    },
    {
      "epoch": 0.9874476987447699,
      "step": 59,
      "total_flos": 0.0,
      "train_loss": 0.5835601434869281,
      "train_runtime": 2693.68,
      "train_samples_per_second": 5.674,
      "train_steps_per_second": 0.022
    }
  ],
  "logging_steps": 10,
  "max_steps": 59,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}