{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9968652037617555,
  "eval_steps": 500,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "learning_rate": 3.125e-08,
      "logits/chosen": -2.1063718795776367,
      "logits/rejected": -2.0172834396362305,
      "logps/chosen": -441.2913818359375,
      "logps/pi_response": -291.7864990234375,
      "logps/ref_response": -291.7864990234375,
      "logps/rejected": -407.91192626953125,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.116103172302246,
      "logits/rejected": -2.0064265727996826,
      "logps/chosen": -276.1221008300781,
      "logps/pi_response": -191.32034301757812,
      "logps/ref_response": -190.91871643066406,
      "logps/rejected": -346.5228576660156,
      "loss": 0.687,
      "rewards/accuracies": 0.5694444179534912,
      "rewards/chosen": -0.010301925241947174,
      "rewards/margins": 0.014795614406466484,
      "rewards/rejected": -0.025097541511058807,
      "step": 10
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -2.0136561393737793,
      "logits/rejected": -1.981844186782837,
      "logps/chosen": -291.6961975097656,
      "logps/pi_response": -205.843505859375,
      "logps/ref_response": -189.8782196044922,
      "logps/rejected": -444.3314514160156,
      "loss": 0.6587,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.355610728263855,
      "rewards/margins": 0.2842535376548767,
      "rewards/rejected": -0.6398642659187317,
      "step": 20
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -1.9893925189971924,
      "logits/rejected": -1.946048378944397,
      "logps/chosen": -291.5216979980469,
      "logps/pi_response": -212.42111206054688,
      "logps/ref_response": -199.7083740234375,
      "logps/rejected": -389.95782470703125,
      "loss": 0.673,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.2551540434360504,
      "rewards/margins": 0.27687785029411316,
      "rewards/rejected": -0.5320318937301636,
      "step": 30
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -2.0515387058258057,
      "logits/rejected": -1.974180817604065,
      "logps/chosen": -295.8572998046875,
      "logps/pi_response": -216.64120483398438,
      "logps/ref_response": -197.79104614257812,
      "logps/rejected": -470.1461486816406,
      "loss": 0.6364,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.25649192929267883,
      "rewards/margins": 0.36993804574012756,
      "rewards/rejected": -0.6264299750328064,
      "step": 40
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -2.0759925842285156,
      "logits/rejected": -1.9234635829925537,
      "logps/chosen": -312.621826171875,
      "logps/pi_response": -232.86474609375,
      "logps/ref_response": -209.6178436279297,
      "logps/rejected": -509.26318359375,
      "loss": 0.625,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.3486739993095398,
      "rewards/margins": 0.5309742093086243,
      "rewards/rejected": -0.8796480894088745,
      "step": 50
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -1.797320008277893,
      "logits/rejected": -1.6790682077407837,
      "logps/chosen": -304.28594970703125,
      "logps/pi_response": -239.77108764648438,
      "logps/ref_response": -206.43423461914062,
      "logps/rejected": -470.97528076171875,
      "loss": 0.6067,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.4016168713569641,
      "rewards/margins": 0.475954532623291,
      "rewards/rejected": -0.8775714635848999,
      "step": 60
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -1.44484281539917,
      "logits/rejected": -1.1385029554367065,
      "logps/chosen": -307.74578857421875,
      "logps/pi_response": -231.57217407226562,
      "logps/ref_response": -184.50869750976562,
      "logps/rejected": -461.8819885253906,
      "loss": 0.589,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.42910051345825195,
      "rewards/margins": 0.6111310124397278,
      "rewards/rejected": -1.040231466293335,
      "step": 70
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -1.2302042245864868,
      "logits/rejected": -0.8724157214164734,
      "logps/chosen": -384.6354064941406,
      "logps/pi_response": -266.6485290527344,
      "logps/ref_response": -202.98727416992188,
      "logps/rejected": -476.89410400390625,
      "loss": 0.5814,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7001193165779114,
      "rewards/margins": 0.4296953082084656,
      "rewards/rejected": -1.1298145055770874,
      "step": 80
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -1.0443007946014404,
      "logits/rejected": -0.5559221506118774,
      "logps/chosen": -355.37078857421875,
      "logps/pi_response": -265.6754150390625,
      "logps/ref_response": -202.15640258789062,
      "logps/rejected": -486.13507080078125,
      "loss": 0.5948,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6906540989875793,
      "rewards/margins": 0.4475804269313812,
      "rewards/rejected": -1.1382344961166382,
      "step": 90
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -0.894809901714325,
      "logits/rejected": -0.34124261140823364,
      "logps/chosen": -358.05755615234375,
      "logps/pi_response": -283.3772277832031,
      "logps/ref_response": -207.0720672607422,
      "logps/rejected": -521.771728515625,
      "loss": 0.5624,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7900195121765137,
      "rewards/margins": 0.6714037656784058,
      "rewards/rejected": -1.4614231586456299,
      "step": 100
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -0.8867988586425781,
      "logits/rejected": -0.3483342230319977,
      "logps/chosen": -380.3785095214844,
      "logps/pi_response": -293.0396728515625,
      "logps/ref_response": -201.5030975341797,
      "logps/rejected": -526.4119873046875,
      "loss": 0.5833,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8987709283828735,
      "rewards/margins": 0.5739961266517639,
      "rewards/rejected": -1.4727669954299927,
      "step": 110
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -0.8515814542770386,
      "logits/rejected": -0.5452559590339661,
      "logps/chosen": -343.3797912597656,
      "logps/pi_response": -272.0332336425781,
      "logps/ref_response": -198.11781311035156,
      "logps/rejected": -536.7486572265625,
      "loss": 0.5449,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.7609699368476868,
      "rewards/margins": 0.6519715785980225,
      "rewards/rejected": -1.412941575050354,
      "step": 120
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -0.8249815702438354,
      "logits/rejected": -0.48517799377441406,
      "logps/chosen": -350.73858642578125,
      "logps/pi_response": -274.28204345703125,
      "logps/ref_response": -193.91168212890625,
      "logps/rejected": -545.8704223632812,
      "loss": 0.5521,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.7474575042724609,
      "rewards/margins": 0.7682065963745117,
      "rewards/rejected": -1.5156641006469727,
      "step": 130
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": -0.6546958088874817,
      "logits/rejected": -0.09045759588479996,
      "logps/chosen": -367.89312744140625,
      "logps/pi_response": -284.0288391113281,
      "logps/ref_response": -196.39334106445312,
      "logps/rejected": -535.0501708984375,
      "loss": 0.5633,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.8734307289123535,
      "rewards/margins": 0.7547620534896851,
      "rewards/rejected": -1.6281929016113281,
      "step": 140
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -0.8392259478569031,
      "logits/rejected": -0.3418545424938202,
      "logps/chosen": -361.2017517089844,
      "logps/pi_response": -292.3738708496094,
      "logps/ref_response": -211.84817504882812,
      "logps/rejected": -521.71875,
      "loss": 0.5783,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.8251138925552368,
      "rewards/margins": 0.5152265429496765,
      "rewards/rejected": -1.340340495109558,
      "step": 150
    },
    {
      "epoch": 1.0,
      "step": 159,
      "total_flos": 0.0,
      "train_loss": 0.5938605962309448,
      "train_runtime": 4590.1424,
      "train_samples_per_second": 4.44,
      "train_steps_per_second": 0.035
    }
  ],
  "logging_steps": 10,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}