{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.8666666666666667,
  "global_step": 14,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0,
      "logps_train/chosen": -100.49485778808594,
      "logps_train/ref_chosen": -100.5,
      "logps_train/ref_rejected": -105.0,
      "logps_train/rejected": -104.80752563476562,
      "rewards_train/accuracies": 0.46875,
      "rewards_train/chosen": -0.020498279482126236,
      "rewards_train/margins": -0.021032828837633133,
      "rewards_train/rejected": 0.000534549355506897,
      "step": 0
    },
    {
      "epoch": 0,
      "logps_train/chosen": -89.90950012207031,
      "logps_train/ref_chosen": -90.0,
      "logps_train/ref_rejected": -101.0,
      "logps_train/rejected": -100.86872863769531,
      "rewards_train/accuracies": 0.5,
      "rewards_train/chosen": 0.0191262885928154,
      "rewards_train/margins": 0.0020734891295433044,
      "rewards_train/rejected": 0.017052799463272095,
      "step": 0
    },
    {
      "epoch": 0.13,
      "learning_rate": 5e-06,
      "loss": 0.7043,
      "step": 1
    },
    {
      "epoch": 0.13,
      "logps_train/chosen": -104.38166809082031,
      "logps_train/ref_chosen": -104.5,
      "logps_train/ref_rejected": -98.0,
      "logps_train/rejected": -97.89907836914062,
      "rewards_train/accuracies": 0.515625,
      "rewards_train/chosen": 0.03059956058859825,
      "rewards_train/margins": 0.004916800186038017,
      "rewards_train/rejected": 0.025682760402560234,
      "step": 1
    },
    {
      "epoch": 0.13,
      "logps_train/chosen": -94.88069152832031,
      "logps_train/ref_chosen": -95.0,
      "logps_train/ref_rejected": -95.0,
      "logps_train/rejected": -95.29293823242188,
      "rewards_train/accuracies": 0.546875,
      "rewards_train/chosen": -0.009438544511795044,
      "rewards_train/margins": -0.020688317716121674,
      "rewards_train/rejected": 0.01124977320432663,
      "step": 1
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.927354543565131e-06,
      "loss": 0.7061,
      "step": 2
    },
    {
      "epoch": 0.27,
      "logps_train/chosen": -82.45372009277344,
      "logps_train/ref_chosen": -82.5,
      "logps_train/ref_rejected": -89.0,
      "logps_train/rejected": -89.00653839111328,
      "rewards_train/accuracies": 0.5625,
      "rewards_train/chosen": 0.0465165413916111,
      "rewards_train/margins": 0.0453295623883605,
      "rewards_train/rejected": 0.001186979003250599,
      "step": 2
    },
    {
      "epoch": 0.27,
      "logps_train/chosen": -100.2186508178711,
      "logps_train/ref_chosen": -100.0,
      "logps_train/ref_rejected": -106.0,
      "logps_train/rejected": -106.00666809082031,
      "rewards_train/accuracies": 0.59375,
      "rewards_train/chosen": -0.029247179627418518,
      "rewards_train/margins": -0.0325072705745697,
      "rewards_train/rejected": 0.003260090947151184,
      "step": 2
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.7136400641330245e-06,
      "loss": 0.6969,
      "step": 3
    },
    {
      "epoch": 0.4,
      "logps_train/chosen": -85.843994140625,
      "logps_train/ref_chosen": -86.0,
      "logps_train/ref_rejected": -90.0,
      "logps_train/rejected": -90.01055908203125,
      "rewards_train/accuracies": 0.5625,
      "rewards_train/chosen": 0.0008565783500671387,
      "rewards_train/margins": -0.012054374441504478,
      "rewards_train/rejected": 0.012910952791571617,
      "step": 3
    },
    {
      "epoch": 0.4,
      "logps_train/chosen": -92.26002502441406,
      "logps_train/ref_chosen": -92.0,
      "logps_train/ref_rejected": -77.5,
      "logps_train/rejected": -77.61531829833984,
      "rewards_train/accuracies": 0.546875,
      "rewards_train/chosen": 0.024773243814706802,
      "rewards_train/margins": 0.04251495748758316,
      "rewards_train/rejected": -0.017741713672876358,
      "step": 3
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.3712768704277535e-06,
      "loss": 0.6936,
      "step": 4
    },
    {
      "epoch": 0.53,
      "logps_train/chosen": -88.51109313964844,
      "logps_train/ref_chosen": -88.5,
      "logps_train/ref_rejected": -91.5,
      "logps_train/rejected": -91.55830383300781,
      "rewards_train/accuracies": 0.53125,
      "rewards_train/chosen": 0.044210705906152725,
      "rewards_train/margins": 0.0598596166819334,
      "rewards_train/rejected": -0.015648910775780678,
      "step": 4
    },
    {
      "epoch": 0.53,
      "logps_train/chosen": -104.84616088867188,
      "logps_train/ref_chosen": -105.0,
      "logps_train/ref_rejected": -104.5,
      "logps_train/rejected": -104.39971923828125,
      "rewards_train/accuracies": 0.515625,
      "rewards_train/chosen": 0.04029738903045654,
      "rewards_train/margins": 0.04008368402719498,
      "rewards_train/rejected": 0.00021370500326156616,
      "step": 4
    },
    {
      "epoch": 0.67,
      "learning_rate": 3.92016186682789e-06,
      "loss": 0.6761,
      "step": 5
    },
    {
      "epoch": 0.67,
      "logps_train/chosen": -109.26580047607422,
      "logps_train/ref_chosen": -109.0,
      "logps_train/ref_rejected": -99.5,
      "logps_train/rejected": -99.34159851074219,
      "rewards_train/accuracies": 0.40625,
      "rewards_train/chosen": 0.017489098012447357,
      "rewards_train/margins": -0.0021405071020126343,
      "rewards_train/rejected": 0.01962960511445999,
      "step": 5
    },
    {
      "epoch": 0.67,
      "logps_train/chosen": -82.48857116699219,
      "logps_train/ref_chosen": -82.5,
      "logps_train/ref_rejected": -80.0,
      "logps_train/rejected": -79.8382568359375,
      "rewards_train/accuracies": 0.46875,
      "rewards_train/chosen": 0.05613037571310997,
      "rewards_train/margins": 0.01352725550532341,
      "rewards_train/rejected": 0.04260312020778656,
      "step": 5
    },
    {
      "epoch": 0.8,
      "learning_rate": 3.386512217606339e-06,
      "loss": 0.6952,
      "step": 6
    },
    {
      "epoch": 0.8,
      "logps_train/chosen": -92.07105255126953,
      "logps_train/ref_chosen": -92.0,
      "logps_train/ref_rejected": -89.5,
      "logps_train/rejected": -89.48802185058594,
      "rewards_train/accuracies": 0.53125,
      "rewards_train/chosen": 0.07213925570249557,
      "rewards_train/margins": -0.003675796091556549,
      "rewards_train/rejected": 0.07581505179405212,
      "step": 6
    },
    {
      "epoch": 0.8,
      "logps_train/chosen": -88.25656127929688,
      "logps_train/ref_chosen": -88.5,
      "logps_train/ref_rejected": -94.0,
      "logps_train/rejected": -93.6957778930664,
      "rewards_train/accuracies": 0.46875,
      "rewards_train/chosen": 0.025041330605745316,
      "rewards_train/margins": -0.009836137294769287,
      "rewards_train/rejected": 0.0348774679005146,
      "step": 6
    },
    {
      "epoch": 0.93,
      "learning_rate": 2.8013417006383078e-06,
      "loss": 0.7033,
      "step": 7
    },
    {
      "epoch": 0.93,
      "logps_train/chosen": -106.97582244873047,
      "logps_train/ref_chosen": -107.0,
      "logps_train/ref_rejected": -104.0,
      "logps_train/rejected": -104.00101470947266,
      "rewards_train/accuracies": 0.78125,
      "rewards_train/chosen": 0.1645529866218567,
      "rewards_train/margins": 0.235374353826046,
      "rewards_train/rejected": -0.0708213672041893,
      "step": 7
    },
    {
      "epoch": 0.93,
      "logps_train/chosen": -95.53459167480469,
      "logps_train/ref_chosen": -96.0,
      "logps_train/ref_rejected": -90.5,
      "logps_train/rejected": -90.36405944824219,
      "rewards_train/accuracies": 0.734375,
      "rewards_train/chosen": 0.18082404136657715,
      "rewards_train/margins": 0.21460064873099327,
      "rewards_train/rejected": -0.03377660736441612,
      "step": 7
    },
    {
      "epoch": 1.07,
      "learning_rate": 2.1986582993616926e-06,
      "loss": 0.5967,
      "step": 8
    },
    {
      "epoch": 1.07,
      "logps_train/chosen": -103.63533020019531,
      "logps_train/ref_chosen": -104.0,
      "logps_train/ref_rejected": -91.0,
      "logps_train/rejected": -91.11985778808594,
      "rewards_train/accuracies": 0.6875,
      "rewards_train/chosen": 0.211873859167099,
      "rewards_train/margins": 0.20722994953393936,
      "rewards_train/rejected": 0.0046439096331596375,
      "step": 8
    },
    {
      "epoch": 1.07,
      "logps_train/chosen": -89.88359069824219,
      "logps_train/ref_chosen": -90.0,
      "logps_train/ref_rejected": -91.0,
      "logps_train/rejected": -91.22651672363281,
      "rewards_train/accuracies": 0.71875,
      "rewards_train/chosen": 0.1387726217508316,
      "rewards_train/margins": 0.168781116604805,
      "rewards_train/rejected": -0.03000849485397339,
      "step": 8
    },
    {
      "epoch": 1.2,
      "learning_rate": 1.613487782393661e-06,
      "loss": 0.6165,
      "step": 9
    },
    {
      "epoch": 1.2,
      "logps_train/chosen": -99.79483795166016,
      "logps_train/ref_chosen": -100.0,
      "logps_train/ref_rejected": -91.5,
      "logps_train/rejected": -91.58244323730469,
      "rewards_train/accuracies": 0.78125,
      "rewards_train/chosen": 0.2085389792919159,
      "rewards_train/margins": 0.2784496992826462,
      "rewards_train/rejected": -0.06991071999073029,
      "step": 9
    },
    {
      "epoch": 1.2,
      "logps_train/chosen": -97.40536499023438,
      "logps_train/ref_chosen": -98.0,
      "logps_train/ref_rejected": -95.5,
      "logps_train/rejected": -95.38594055175781,
      "rewards_train/accuracies": 0.765625,
      "rewards_train/chosen": 0.16743528842926025,
      "rewards_train/margins": 0.1775425188243389,
      "rewards_train/rejected": -0.010107230395078659,
      "step": 9
    },
    {
      "epoch": 1.33,
      "learning_rate": 1.079838133172111e-06,
      "loss": 0.5951,
      "step": 10
    },
    {
      "epoch": 1.33,
      "logps_train/chosen": -99.17311096191406,
      "logps_train/ref_chosen": -99.5,
      "logps_train/ref_rejected": -97.0,
      "logps_train/rejected": -96.72930908203125,
      "rewards_train/accuracies": 0.703125,
      "rewards_train/chosen": 0.21544356644153595,
      "rewards_train/margins": 0.18458961695432663,
      "rewards_train/rejected": 0.03085394948720932,
      "step": 10
    },
    {
      "epoch": 1.33,
      "logps_train/chosen": -90.94091796875,
      "logps_train/ref_chosen": -91.0,
      "logps_train/ref_rejected": -97.5,
      "logps_train/rejected": -97.65658569335938,
      "rewards_train/accuracies": 0.8125,
      "rewards_train/chosen": 0.21173310279846191,
      "rewards_train/margins": 0.18907413445413113,
      "rewards_train/rejected": 0.022658968344330788,
      "step": 10
    },
    {
      "epoch": 1.47,
      "learning_rate": 6.28723129572247e-07,
      "loss": 0.6138,
      "step": 11
    },
    {
      "epoch": 1.47,
      "logps_train/chosen": -96.39669799804688,
      "logps_train/ref_chosen": -97.0,
      "logps_train/ref_rejected": -99.0,
      "logps_train/rejected": -98.8724136352539,
      "rewards_train/accuracies": 0.8125,
      "rewards_train/chosen": 0.16395366191864014,
      "rewards_train/margins": 0.25006232410669327,
      "rewards_train/rejected": -0.08610866218805313,
      "step": 11
    },
    {
      "epoch": 1.47,
      "logps_train/chosen": -80.8447036743164,
      "logps_train/ref_chosen": -81.0,
      "logps_train/ref_rejected": -90.0,
      "logps_train/rejected": -90.13240814208984,
      "rewards_train/accuracies": 0.734375,
      "rewards_train/chosen": 0.13546136021614075,
      "rewards_train/margins": 0.16997329890727997,
      "rewards_train/rejected": -0.03451193869113922,
      "step": 11
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.8635993586697555e-07,
      "loss": 0.6012,
      "step": 12
    },
    {
      "epoch": 1.6,
      "logps_train/chosen": -89.55223846435547,
      "logps_train/ref_chosen": -90.0,
      "logps_train/ref_rejected": -96.0,
      "logps_train/rejected": -95.89089965820312,
      "rewards_train/accuracies": 0.84375,
      "rewards_train/chosen": 0.20703516900539398,
      "rewards_train/margins": 0.23602662980556488,
      "rewards_train/rejected": -0.0289914608001709,
      "step": 12
    },
    {
      "epoch": 1.6,
      "logps_train/chosen": -82.7729263305664,
      "logps_train/ref_chosen": -83.0,
      "logps_train/ref_rejected": -85.0,
      "logps_train/rejected": -85.27727508544922,
      "rewards_train/accuracies": 0.796875,
      "rewards_train/chosen": 0.17896729707717896,
      "rewards_train/margins": 0.2392353191971779,
      "rewards_train/rejected": -0.06026802211999893,
      "step": 12
    },
    {
      "epoch": 1.73,
      "learning_rate": 7.264545643486997e-08,
      "loss": 0.5902,
      "step": 13
    },
    {
      "epoch": 1.73,
      "logps_train/chosen": -104.69845581054688,
      "logps_train/ref_chosen": -105.0,
      "logps_train/ref_rejected": -101.0,
      "logps_train/rejected": -100.98780822753906,
      "rewards_train/accuracies": 0.796875,
      "rewards_train/chosen": 0.19716176390647888,
      "rewards_train/margins": 0.24550895392894745,
      "rewards_train/rejected": -0.04834719002246857,
      "step": 13
    },
    {
      "epoch": 1.73,
      "logps_train/chosen": -90.20387268066406,
      "logps_train/ref_chosen": -91.0,
      "logps_train/ref_rejected": -85.0,
      "logps_train/rejected": -85.12068939208984,
      "rewards_train/accuracies": 0.65625,
      "rewards_train/chosen": 0.1614937037229538,
      "rewards_train/margins": 0.1774044744670391,
      "rewards_train/rejected": -0.015910770744085312,
      "step": 13
    },
    {
      "epoch": 1.87,
      "learning_rate": 0.0,
      "loss": 0.6056,
      "step": 14
    },
    {
      "epoch": 1.87,
      "step": 14,
      "total_flos": 0.0,
      "train_loss": 0.6496244881834302,
      "train_runtime": 164.1376,
      "train_samples_per_second": 11.076,
      "train_steps_per_second": 0.085
    }
  ],
  "max_steps": 14,
  "num_train_epochs": 2,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}