{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984301412872841,
  "eval_steps": 500,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "learning_rate": 1.875e-08,
      "logits/chosen": 0.15145538747310638,
      "logits/rejected": 0.6641070246696472,
      "logps/chosen": -279.5725402832031,
      "logps/pi_response": -165.03363037109375,
      "logps/ref_response": -165.03363037109375,
      "logps/rejected": -323.2174072265625,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.875e-07,
      "logits/chosen": 0.636666476726532,
      "logits/rejected": 0.9183499217033386,
      "logps/chosen": -273.43743896484375,
      "logps/pi_response": -161.16036987304688,
      "logps/ref_response": -161.25872802734375,
      "logps/rejected": -399.26593017578125,
      "loss": 0.6924,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0005922508426010609,
      "rewards/margins": 0.0008381960215047002,
      "rewards/rejected": -0.0002459452080074698,
      "step": 10
    },
    {
      "epoch": 0.13,
      "learning_rate": 2.9942119880575817e-07,
      "logits/chosen": 0.5393803715705872,
      "logits/rejected": 0.8416255712509155,
      "logps/chosen": -294.79083251953125,
      "logps/pi_response": -178.3276824951172,
      "logps/ref_response": -178.49508666992188,
      "logps/rejected": -425.2745056152344,
      "loss": 0.6768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.01799004338681698,
      "rewards/margins": 0.04210059717297554,
      "rewards/rejected": -0.06009063869714737,
      "step": 20
    },
    {
      "epoch": 0.19,
      "learning_rate": 2.929608750821129e-07,
      "logits/chosen": 0.5195704698562622,
      "logits/rejected": 1.0099811553955078,
      "logps/chosen": -289.0838928222656,
      "logps/pi_response": -160.77481079101562,
      "logps/ref_response": -160.6930694580078,
      "logps/rejected": -437.66064453125,
      "loss": 0.6228,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.12752258777618408,
      "rewards/margins": 0.18380290269851685,
      "rewards/rejected": -0.3113254904747009,
      "step": 30
    },
    {
      "epoch": 0.25,
      "learning_rate": 2.7962832564252725e-07,
      "logits/chosen": 0.5080239772796631,
      "logits/rejected": 0.923050045967102,
      "logps/chosen": -338.06256103515625,
      "logps/pi_response": -181.06967163085938,
      "logps/ref_response": -179.89529418945312,
      "logps/rejected": -483.7896423339844,
      "loss": 0.5846,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2662404775619507,
      "rewards/margins": 0.3624354302883148,
      "rewards/rejected": -0.6286758780479431,
      "step": 40
    },
    {
      "epoch": 0.31,
      "learning_rate": 2.6006445513357056e-07,
      "logits/chosen": 0.6372388005256653,
      "logits/rejected": 0.9440908432006836,
      "logps/chosen": -354.6672058105469,
      "logps/pi_response": -182.6870880126953,
      "logps/ref_response": -174.0111083984375,
      "logps/rejected": -507.6558532714844,
      "loss": 0.5679,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6286555528640747,
      "rewards/margins": 0.5696737170219421,
      "rewards/rejected": -1.1983293294906616,
      "step": 50
    },
    {
      "epoch": 0.38,
      "learning_rate": 2.3520971200967334e-07,
      "logits/chosen": 0.6604863405227661,
      "logits/rejected": 1.0091063976287842,
      "logps/chosen": -351.96807861328125,
      "logps/pi_response": -181.9377899169922,
      "logps/ref_response": -171.8192138671875,
      "logps/rejected": -482.25830078125,
      "loss": 0.569,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.5858457088470459,
      "rewards/margins": 0.5204802751541138,
      "rewards/rejected": -1.1063258647918701,
      "step": 60
    },
    {
      "epoch": 0.44,
      "learning_rate": 2.0625888054143427e-07,
      "logits/chosen": 0.594717800617218,
      "logits/rejected": 0.9649769067764282,
      "logps/chosen": -328.1702880859375,
      "logps/pi_response": -193.00442504882812,
      "logps/ref_response": -186.24789428710938,
      "logps/rejected": -546.3258666992188,
      "loss": 0.5604,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.41636139154434204,
      "rewards/margins": 0.755541980266571,
      "rewards/rejected": -1.171903371810913,
      "step": 70
    },
    {
      "epoch": 0.5,
      "learning_rate": 1.7460364672965327e-07,
      "logits/chosen": 0.6635435223579407,
      "logits/rejected": 1.0644786357879639,
      "logps/chosen": -291.11651611328125,
      "logps/pi_response": -159.24737548828125,
      "logps/ref_response": -152.75279235839844,
      "logps/rejected": -501.09100341796875,
      "loss": 0.5291,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.4029166102409363,
      "rewards/margins": 0.6375669836997986,
      "rewards/rejected": -1.0404835939407349,
      "step": 80
    },
    {
      "epoch": 0.57,
      "learning_rate": 1.4176569902035086e-07,
      "logits/chosen": 0.6440488696098328,
      "logits/rejected": 1.0133155584335327,
      "logps/chosen": -343.33245849609375,
      "logps/pi_response": -168.49266052246094,
      "logps/ref_response": -159.75521850585938,
      "logps/rejected": -521.93359375,
      "loss": 0.5293,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.4756793975830078,
      "rewards/margins": 0.6875613927841187,
      "rewards/rejected": -1.1632407903671265,
      "step": 90
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.0932357971453743e-07,
      "logits/chosen": 0.7471474409103394,
      "logits/rejected": 1.000234842300415,
      "logps/chosen": -335.2603759765625,
      "logps/pi_response": -164.00418090820312,
      "logps/ref_response": -157.03810119628906,
      "logps/rejected": -535.8643188476562,
      "loss": 0.5481,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.49526938796043396,
      "rewards/margins": 0.6929213404655457,
      "rewards/rejected": -1.1881908178329468,
      "step": 100
    },
    {
      "epoch": 0.69,
      "learning_rate": 7.883680337481599e-08,
      "logits/chosen": 0.6913807988166809,
      "logits/rejected": 0.9721935391426086,
      "logps/chosen": -321.69256591796875,
      "logps/pi_response": -177.0824432373047,
      "logps/ref_response": -168.18710327148438,
      "logps/rejected": -506.9715881347656,
      "loss": 0.5309,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.42671942710876465,
      "rewards/margins": 0.6666349172592163,
      "rewards/rejected": -1.093354344367981,
      "step": 110
    },
    {
      "epoch": 0.75,
      "learning_rate": 5.177088990820725e-08,
      "logits/chosen": 0.48052477836608887,
      "logits/rejected": 0.8082226514816284,
      "logps/chosen": -351.45391845703125,
      "logps/pi_response": -192.8612060546875,
      "logps/ref_response": -185.12771606445312,
      "logps/rejected": -559.8470458984375,
      "loss": 0.5239,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.4745051860809326,
      "rewards/margins": 0.7649967670440674,
      "rewards/rejected": -1.239501953125,
      "step": 120
    },
    {
      "epoch": 0.82,
      "learning_rate": 2.942691603548416e-08,
      "logits/chosen": 0.5350357890129089,
      "logits/rejected": 1.0080773830413818,
      "logps/chosen": -345.943115234375,
      "logps/pi_response": -193.96786499023438,
      "logps/ref_response": -185.8845977783203,
      "logps/rejected": -552.7396240234375,
      "loss": 0.5196,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.4510205388069153,
      "rewards/margins": 0.8083527684211731,
      "rewards/rejected": -1.2593733072280884,
      "step": 130
    },
    {
      "epoch": 0.88,
      "learning_rate": 1.2878971655412513e-08,
      "logits/chosen": 0.5684305429458618,
      "logits/rejected": 0.8962447047233582,
      "logps/chosen": -365.9832458496094,
      "logps/pi_response": -200.02487182617188,
      "logps/ref_response": -190.11685180664062,
      "logps/rejected": -579.375732421875,
      "loss": 0.5231,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.5118390321731567,
      "rewards/margins": 0.78752201795578,
      "rewards/rejected": -1.299360990524292,
      "step": 140
    },
    {
      "epoch": 0.94,
      "learning_rate": 2.922527618666465e-09,
      "logits/chosen": 0.5044765472412109,
      "logits/rejected": 0.8951088786125183,
      "logps/chosen": -339.7613830566406,
      "logps/pi_response": -178.6360626220703,
      "logps/ref_response": -168.48829650878906,
      "logps/rejected": -534.9398193359375,
      "loss": 0.5416,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5073550343513489,
      "rewards/margins": 0.6938044428825378,
      "rewards/rejected": -1.2011594772338867,
      "step": 150
    },
    {
      "epoch": 1.0,
      "step": 159,
      "total_flos": 0.0,
      "train_loss": 0.5675025166205641,
      "train_runtime": 4333.3313,
      "train_samples_per_second": 4.703,
      "train_steps_per_second": 0.037
    }
  ],
  "logging_steps": 10,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}