{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9968652037617555,
  "eval_steps": 500,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "learning_rate": 3.125e-08,
      "logits/chosen": -2.0781588554382324,
      "logits/rejected": -1.9858250617980957,
      "logps/chosen": -453.2818603515625,
      "logps/pi_response": -300.6171875,
      "logps/ref_response": -300.6171875,
      "logps/rejected": -393.94146728515625,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.102860689163208,
      "logits/rejected": -2.033174753189087,
      "logps/chosen": -286.9140930175781,
      "logps/pi_response": -192.1187744140625,
      "logps/ref_response": -192.19212341308594,
      "logps/rejected": -342.8356018066406,
      "loss": 0.6925,
      "rewards/accuracies": 0.4513888955116272,
      "rewards/chosen": -0.0006262523238547146,
      "rewards/margins": 0.0011568003101274371,
      "rewards/rejected": -0.0017830526921898127,
      "step": 10
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -1.9538593292236328,
      "logits/rejected": -1.9250879287719727,
      "logps/chosen": -318.6830749511719,
      "logps/pi_response": -219.6834716796875,
      "logps/ref_response": -184.64120483398438,
      "logps/rejected": -481.122802734375,
      "loss": 0.6773,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.06833034753799438,
      "rewards/margins": 0.036504171788692474,
      "rewards/rejected": -0.10483451187610626,
      "step": 20
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -1.4626009464263916,
      "logits/rejected": -1.3391458988189697,
      "logps/chosen": -680.2674560546875,
      "logps/pi_response": -502.0758361816406,
      "logps/ref_response": -192.75753784179688,
      "logps/rejected": -882.34033203125,
      "loss": 0.6418,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.39676880836486816,
      "rewards/margins": 0.1544235348701477,
      "rewards/rejected": -0.5511924028396606,
      "step": 30
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -1.0197283029556274,
      "logits/rejected": -0.6147260069847107,
      "logps/chosen": -553.42333984375,
      "logps/pi_response": -358.3141784667969,
      "logps/ref_response": -189.3666534423828,
      "logps/rejected": -931.6701049804688,
      "loss": 0.64,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.2808569073677063,
      "rewards/margins": 0.278983473777771,
      "rewards/rejected": -0.5598403811454773,
      "step": 40
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -0.4955853521823883,
      "logits/rejected": 0.25757771730422974,
      "logps/chosen": -521.3439331054688,
      "logps/pi_response": -355.96917724609375,
      "logps/ref_response": -208.11865234375,
      "logps/rejected": -1041.2548828125,
      "loss": 0.6254,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.23494353890419006,
      "rewards/margins": 0.3884660601615906,
      "rewards/rejected": -0.6234095692634583,
      "step": 50
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": 0.10438306629657745,
      "logits/rejected": 0.6441744565963745,
      "logps/chosen": -657.8551025390625,
      "logps/pi_response": -464.1085510253906,
      "logps/ref_response": -197.88754272460938,
      "logps/rejected": -1158.5789794921875,
      "loss": 0.6116,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.38527947664260864,
      "rewards/margins": 0.38742250204086304,
      "rewards/rejected": -0.7727020382881165,
      "step": 60
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -0.016553742811083794,
      "logits/rejected": 0.8143717050552368,
      "logps/chosen": -652.5902709960938,
      "logps/pi_response": -460.16033935546875,
      "logps/ref_response": -180.25257873535156,
      "logps/rejected": -1217.8525390625,
      "loss": 0.6043,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.3822382688522339,
      "rewards/margins": 0.471721351146698,
      "rewards/rejected": -0.8539595603942871,
      "step": 70
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -0.27121174335479736,
      "logits/rejected": 0.5390032529830933,
      "logps/chosen": -682.1510009765625,
      "logps/pi_response": -454.86712646484375,
      "logps/ref_response": -201.3350372314453,
      "logps/rejected": -1071.9420166015625,
      "loss": 0.6021,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.36673516035079956,
      "rewards/margins": 0.34540173411369324,
      "rewards/rejected": -0.7121368646621704,
      "step": 80
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": 0.09567543864250183,
      "logits/rejected": 1.071440577507019,
      "logps/chosen": -664.4251708984375,
      "logps/pi_response": -515.3280029296875,
      "logps/ref_response": -203.6631317138672,
      "logps/rejected": -1307.692138671875,
      "loss": 0.5966,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3908023238182068,
      "rewards/margins": 0.5106150507926941,
      "rewards/rejected": -0.9014174342155457,
      "step": 90
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": 0.02365037612617016,
      "logits/rejected": 0.7325735092163086,
      "logps/chosen": -522.3504638671875,
      "logps/pi_response": -380.5610656738281,
      "logps/ref_response": -213.95852661132812,
      "logps/rejected": -976.3234252929688,
      "loss": 0.6158,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.24089224636554718,
      "rewards/margins": 0.35071635246276855,
      "rewards/rejected": -0.5916085839271545,
      "step": 100
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": 0.14913706481456757,
      "logits/rejected": 1.051020860671997,
      "logps/chosen": -584.4959106445312,
      "logps/pi_response": -380.89190673828125,
      "logps/ref_response": -197.9862823486328,
      "logps/rejected": -1066.765380859375,
      "loss": 0.6126,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.2975122034549713,
      "rewards/margins": 0.3768185079097748,
      "rewards/rejected": -0.6743307113647461,
      "step": 110
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": 0.6599678993225098,
      "logits/rejected": 1.237850546836853,
      "logps/chosen": -677.4052124023438,
      "logps/pi_response": -448.3097229003906,
      "logps/ref_response": -183.84140014648438,
      "logps/rejected": -1206.931884765625,
      "loss": 0.5783,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4090994894504547,
      "rewards/margins": 0.4326961040496826,
      "rewards/rejected": -0.8417955636978149,
      "step": 120
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": 0.6546910405158997,
      "logits/rejected": 1.1750857830047607,
      "logps/chosen": -641.6511840820312,
      "logps/pi_response": -466.5652770996094,
      "logps/ref_response": -192.61219787597656,
      "logps/rejected": -1306.2027587890625,
      "loss": 0.6047,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.36833977699279785,
      "rewards/margins": 0.5396827459335327,
      "rewards/rejected": -0.9080225229263306,
      "step": 130
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": 0.6576046943664551,
      "logits/rejected": 1.7164825201034546,
      "logps/chosen": -646.2034912109375,
      "logps/pi_response": -518.1434326171875,
      "logps/ref_response": -199.8957977294922,
      "logps/rejected": -1313.4241943359375,
      "loss": 0.587,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.37260332703590393,
      "rewards/margins": 0.556897759437561,
      "rewards/rejected": -0.9295011758804321,
      "step": 140
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": 0.3645651936531067,
      "logits/rejected": 0.9882078170776367,
      "logps/chosen": -661.0691528320312,
      "logps/pi_response": -497.58856201171875,
      "logps/ref_response": -216.212158203125,
      "logps/rejected": -1136.5726318359375,
      "loss": 0.594,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3775670826435089,
      "rewards/margins": 0.37788960337638855,
      "rewards/rejected": -0.7554566264152527,
      "step": 150
    },
    {
      "epoch": 1.0,
      "step": 159,
      "total_flos": 0.0,
      "train_loss": 0.6132173088361632,
      "train_runtime": 4607.0079,
      "train_samples_per_second": 4.423,
      "train_steps_per_second": 0.035
    }
  ],
  "logging_steps": 10,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}