{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984,
  "eval_steps": 100,
  "global_step": 312,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.5625e-07,
      "logits/chosen": -0.8750922679901123,
      "logits/rejected": -0.7979680895805359,
      "logps/chosen": -60.00876235961914,
      "logps/rejected": -56.38082504272461,
      "loss": 0.1482,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5625e-06,
      "logits/chosen": -1.0051445960998535,
      "logits/rejected": -0.9556077122688293,
      "logps/chosen": -30.64621925354004,
      "logps/rejected": -24.19979476928711,
      "loss": 0.1565,
      "rewards/accuracies": 0.02083333395421505,
      "rewards/chosen": -8.15845123725012e-05,
      "rewards/margins": -4.479259951040149e-05,
      "rewards/rejected": -3.679191286209971e-05,
      "step": 10
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.125e-06,
      "logits/chosen": -1.0498403310775757,
      "logits/rejected": -0.992475152015686,
      "logps/chosen": -40.17894744873047,
      "logps/rejected": -31.744924545288086,
      "loss": 0.1421,
      "rewards/accuracies": 0.03125,
      "rewards/chosen": -3.2624266168568283e-06,
      "rewards/margins": -6.817355460952967e-05,
      "rewards/rejected": 6.491113163065165e-05,
      "step": 20
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.6875000000000004e-06,
      "logits/chosen": -1.0504692792892456,
      "logits/rejected": -1.0281661748886108,
      "logps/chosen": -23.273496627807617,
      "logps/rejected": -18.378360748291016,
      "loss": 0.1594,
      "rewards/accuracies": 0.01875000074505806,
      "rewards/chosen": -7.991820166353136e-05,
      "rewards/margins": -0.00010900965571636334,
      "rewards/rejected": 2.909145587182138e-05,
      "step": 30
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.989935734988098e-06,
      "logits/chosen": -1.0432171821594238,
      "logits/rejected": -1.0193238258361816,
      "logps/chosen": -14.588598251342773,
      "logps/rejected": -13.306127548217773,
      "loss": 0.1303,
      "rewards/accuracies": 0.012500000186264515,
      "rewards/chosen": 4.2490195482969284e-05,
      "rewards/margins": -2.1462084987433627e-05,
      "rewards/rejected": 6.39522768324241e-05,
      "step": 40
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.949188496058089e-06,
      "logits/chosen": -1.0302413702011108,
      "logits/rejected": -0.986968994140625,
      "logps/chosen": -9.379155158996582,
      "logps/rejected": -9.404864311218262,
      "loss": 0.1469,
      "rewards/accuracies": 0.02500000037252903,
      "rewards/chosen": 3.0187036827555858e-05,
      "rewards/margins": 4.167082079220563e-05,
      "rewards/rejected": -1.1483787602628581e-05,
      "step": 50
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8776412907378845e-06,
      "logits/chosen": -0.9920421838760376,
      "logits/rejected": -0.9573459625244141,
      "logps/chosen": -17.296560287475586,
      "logps/rejected": -15.45374584197998,
      "loss": 0.1519,
      "rewards/accuracies": 0.012500000186264515,
      "rewards/chosen": 2.950535053969361e-05,
      "rewards/margins": -1.730041731207166e-05,
      "rewards/rejected": 4.680576239479706e-05,
      "step": 60
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7761938666470405e-06,
      "logits/chosen": -1.0576204061508179,
      "logits/rejected": -1.0229989290237427,
      "logps/chosen": -30.714122772216797,
      "logps/rejected": -24.89596939086914,
      "loss": 0.1346,
      "rewards/accuracies": 0.05000000074505806,
      "rewards/chosen": 0.00010937159095192328,
      "rewards/margins": -1.5012978110462427e-05,
      "rewards/rejected": 0.0001243845617864281,
      "step": 70
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.646121984004666e-06,
      "logits/chosen": -1.0434526205062866,
      "logits/rejected": -1.0283691883087158,
      "logps/chosen": -12.912440299987793,
      "logps/rejected": -10.21430778503418,
      "loss": 0.1494,
      "rewards/accuracies": 0.01875000074505806,
      "rewards/chosen": 6.350317562464625e-05,
      "rewards/margins": 3.430190190556459e-05,
      "rewards/rejected": 2.9201270081102848e-05,
      "step": 80
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4890613722044526e-06,
      "logits/chosen": -1.005961298942566,
      "logits/rejected": -0.9507063627243042,
      "logps/chosen": -29.203853607177734,
      "logps/rejected": -24.27176856994629,
      "loss": 0.1563,
      "rewards/accuracies": 0.02500000037252903,
      "rewards/chosen": 0.00019098483608104289,
      "rewards/margins": -0.00010294451203662902,
      "rewards/rejected": 0.00029392936266958714,
      "step": 90
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.3069871595684795e-06,
      "logits/chosen": -1.0817575454711914,
      "logits/rejected": -1.0109978914260864,
      "logps/chosen": -23.601749420166016,
      "logps/rejected": -21.154720306396484,
      "loss": 0.1204,
      "rewards/accuracies": 0.03750000149011612,
      "rewards/chosen": 0.0001659760600887239,
      "rewards/margins": 1.1245434507145546e-05,
      "rewards/rejected": 0.00015473061648663133,
      "step": 100
    },
    {
      "epoch": 0.32,
      "eval_logits/chosen": -0.8344239592552185,
      "eval_logits/rejected": -0.7404021620750427,
      "eval_logps/chosen": -395.97412109375,
      "eval_logps/rejected": -366.70782470703125,
      "eval_loss": 0.05379165709018707,
      "eval_rewards/accuracies": 0.4399999976158142,
      "eval_rewards/chosen": 0.0003311456530354917,
      "eval_rewards/margins": -0.0001964901020983234,
      "eval_rewards/rejected": 0.0005276357405818999,
      "eval_runtime": 545.7797,
      "eval_samples_per_second": 3.664,
      "eval_steps_per_second": 0.916,
      "step": 100
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.102189034962561e-06,
      "logits/chosen": -1.0390229225158691,
      "logits/rejected": -1.025102972984314,
      "logps/chosen": -21.16196060180664,
      "logps/rejected": -22.110755920410156,
      "loss": 0.1488,
      "rewards/accuracies": 0.03125,
      "rewards/chosen": 0.00030253888689912856,
      "rewards/margins": 5.533280273084529e-05,
      "rewards/rejected": 0.0002472061023581773,
      "step": 110
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.8772424536302565e-06,
      "logits/chosen": -1.029483437538147,
      "logits/rejected": -0.9574755430221558,
      "logps/chosen": -19.494661331176758,
      "logps/rejected": -17.480262756347656,
      "loss": 0.145,
      "rewards/accuracies": 0.04374999925494194,
      "rewards/chosen": 0.0003196864854544401,
      "rewards/margins": 0.00018756816280074418,
      "rewards/rejected": 0.00013211832265369594,
      "step": 120
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.634976249348867e-06,
      "logits/chosen": -1.107649564743042,
      "logits/rejected": -1.076535701751709,
      "logps/chosen": -11.74854850769043,
      "logps/rejected": -9.875129699707031,
      "loss": 0.1365,
      "rewards/accuracies": 0.01875000074505806,
      "rewards/chosen": 0.00019255562801845372,
      "rewards/margins": 3.773098433157429e-05,
      "rewards/rejected": 0.0001548246364109218,
      "step": 130
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3784370602033572e-06,
      "logits/chosen": -1.0260846614837646,
      "logits/rejected": -0.9438373446464539,
      "logps/chosen": -17.046716690063477,
      "logps/rejected": -12.897377014160156,
      "loss": 0.1655,
      "rewards/accuracies": 0.03125,
      "rewards/chosen": 0.00021877660765312612,
      "rewards/margins": 0.00010056849714601412,
      "rewards/rejected": 0.00011820811778306961,
      "step": 140
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1108510153447352e-06,
      "logits/chosen": -1.0065504312515259,
      "logits/rejected": -0.9879522323608398,
      "logps/chosen": -14.441717147827148,
      "logps/rejected": -11.689096450805664,
      "loss": 0.1525,
      "rewards/accuracies": 0.02500000037252903,
      "rewards/chosen": 0.00024849834153428674,
      "rewards/margins": 9.472499368712306e-05,
      "rewards/rejected": 0.0001537733623990789,
      "step": 150
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.835583164544139e-06,
      "logits/chosen": -1.0789475440979004,
      "logits/rejected": -1.0655437707901,
      "logps/chosen": -14.895776748657227,
      "logps/rejected": -14.265124320983887,
      "loss": 0.1408,
      "rewards/accuracies": 0.02500000037252903,
      "rewards/chosen": 0.00023004882677923888,
      "rewards/margins": -1.2942269677296281e-05,
      "rewards/rejected": 0.0002429911110084504,
      "step": 160
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.556095160739513e-06,
      "logits/chosen": -1.059157371520996,
      "logits/rejected": -0.9842355847358704,
      "logps/chosen": -23.611387252807617,
      "logps/rejected": -22.636394500732422,
      "loss": 0.1427,
      "rewards/accuracies": 0.03125,
      "rewards/chosen": 0.00042936400859616697,
      "rewards/margins": 1.9614741177065298e-05,
      "rewards/rejected": 0.0004097492783330381,
      "step": 170
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2759017277414165e-06,
      "logits/chosen": -1.0258867740631104,
      "logits/rejected": -0.9551021456718445,
      "logps/chosen": -26.574390411376953,
      "logps/rejected": -17.66288948059082,
      "loss": 0.1366,
      "rewards/accuracies": 0.02500000037252903,
      "rewards/chosen": 0.0002309099363628775,
      "rewards/margins": -0.00013474392471835017,
      "rewards/rejected": 0.00036565386108122766,
      "step": 180
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9985264605418185e-06,
      "logits/chosen": -1.0549209117889404,
      "logits/rejected": -1.0101747512817383,
      "logps/chosen": -13.79809284210205,
      "logps/rejected": -12.155397415161133,
      "loss": 0.1457,
      "rewards/accuracies": 0.012500000186264515,
      "rewards/chosen": 0.00018917236593551934,
      "rewards/margins": -1.1611557965807151e-05,
      "rewards/rejected": 0.00020078392117284238,
      "step": 190
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7274575140626318e-06,
      "logits/chosen": -1.0925042629241943,
      "logits/rejected": -1.0523203611373901,
      "logps/chosen": -22.70647621154785,
      "logps/rejected": -21.175107955932617,
      "loss": 0.1673,
      "rewards/accuracies": 0.02500000037252903,
      "rewards/chosen": 0.00025192610337398946,
      "rewards/margins": 6.463247700594366e-06,
      "rewards/rejected": 0.0002454628120176494,
      "step": 200
    },
    {
      "epoch": 0.64,
      "eval_logits/chosen": -0.8413296937942505,
      "eval_logits/rejected": -0.7451605796813965,
      "eval_logps/chosen": -394.95245361328125,
      "eval_logps/rejected": -365.8174743652344,
      "eval_loss": 0.05375674366950989,
      "eval_rewards/accuracies": 0.4465000033378601,
      "eval_rewards/chosen": 0.0013527950504794717,
      "eval_rewards/margins": -6.51527734589763e-05,
      "eval_rewards/rejected": 0.0014179478166624904,
      "eval_runtime": 545.9031,
      "eval_samples_per_second": 3.664,
      "eval_steps_per_second": 0.916,
      "step": 200
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.466103737583699e-06,
      "logits/chosen": -1.0759646892547607,
      "logits/rejected": -1.0574228763580322,
      "logps/chosen": -21.28565216064453,
      "logps/rejected": -18.393810272216797,
      "loss": 0.1471,
      "rewards/accuracies": 0.01875000074505806,
      "rewards/chosen": 0.0002163530298275873,
      "rewards/margins": -0.00010527259291848168,
      "rewards/rejected": 0.0003216256445739418,
      "step": 210
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.217751806485235e-06,
      "logits/chosen": -1.036292314529419,
      "logits/rejected": -0.9575215578079224,
      "logps/chosen": -19.784107208251953,
      "logps/rejected": -16.992115020751953,
      "loss": 0.1377,
      "rewards/accuracies": 0.03125,
      "rewards/chosen": 0.00023899655207060277,
      "rewards/margins": 5.950551712885499e-05,
      "rewards/rejected": 0.000179491049493663,
      "step": 220
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.855248903979505e-07,
      "logits/chosen": -1.042037844657898,
      "logits/rejected": -1.0194261074066162,
      "logps/chosen": -29.382129669189453,
      "logps/rejected": -24.937179565429688,
      "loss": 0.1457,
      "rewards/accuracies": 0.05000000074505806,
      "rewards/chosen": 0.0005346836405806243,
      "rewards/margins": 4.8290185077348724e-05,
      "rewards/rejected": 0.0004863934591412544,
      "step": 230
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.723433775328385e-07,
      "logits/chosen": -1.0891209840774536,
      "logits/rejected": -1.0485570430755615,
      "logps/chosen": -24.730152130126953,
      "logps/rejected": -23.654470443725586,
      "loss": 0.157,
      "rewards/accuracies": 0.03125,
      "rewards/chosen": 0.00022753537632524967,
      "rewards/margins": -0.00011534886289155111,
      "rewards/rejected": 0.0003428841882850975,
      "step": 240
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.808881491049723e-07,
      "logits/chosen": -1.1149171590805054,
      "logits/rejected": -1.0429071187973022,
      "logps/chosen": -38.763633728027344,
      "logps/rejected": -27.425064086914062,
      "loss": 0.1685,
      "rewards/accuracies": 0.05000000074505806,
      "rewards/chosen": 0.0005215436685830355,
      "rewards/margins": 2.6613433874445036e-05,
      "rewards/rejected": 0.0004949303111061454,
      "step": 250
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1356686569674344e-07,
      "logits/chosen": -1.0859912633895874,
      "logits/rejected": -1.0464859008789062,
      "logps/chosen": -27.60003662109375,
      "logps/rejected": -23.53853988647461,
      "loss": 0.1439,
      "rewards/accuracies": 0.03125,
      "rewards/chosen": 0.000496146094519645,
      "rewards/margins": 2.9071514290990308e-05,
      "rewards/rejected": 0.00046707456931471825,
      "step": 260
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.7248368952908055e-07,
      "logits/chosen": -1.0678870677947998,
      "logits/rejected": -1.0435296297073364,
      "logps/chosen": -32.16819763183594,
      "logps/rejected": -28.180551528930664,
      "loss": 0.1724,
      "rewards/accuracies": 0.03750000149011612,
      "rewards/chosen": 0.0004276956315152347,
      "rewards/margins": 2.2596535927732475e-05,
      "rewards/rejected": 0.00040509915561415255,
      "step": 270
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.59412823400657e-07,
      "logits/chosen": -1.0548503398895264,
      "logits/rejected": -1.0008891820907593,
      "logps/chosen": -25.87237548828125,
      "logps/rejected": -24.495941162109375,
      "loss": 0.1347,
      "rewards/accuracies": 0.03750000149011612,
      "rewards/chosen": 0.0003715341736096889,
      "rewards/margins": -5.351124127628282e-05,
      "rewards/rejected": 0.00042504543671384454,
      "step": 280
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.577619905828281e-08,
      "logits/chosen": -1.0749586820602417,
      "logits/rejected": -1.0013477802276611,
      "logps/chosen": -24.45828628540039,
      "logps/rejected": -17.479082107543945,
      "loss": 0.1346,
      "rewards/accuracies": 0.03125,
      "rewards/chosen": 0.0003076163411606103,
      "rewards/margins": 9.37794684432447e-05,
      "rewards/rejected": 0.00021383687271736562,
      "step": 290
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.262559558016325e-08,
      "logits/chosen": -1.0697089433670044,
      "logits/rejected": -1.0091029405593872,
      "logps/chosen": -17.250530242919922,
      "logps/rejected": -16.223955154418945,
      "loss": 0.1395,
      "rewards/accuracies": 0.01875000074505806,
      "rewards/chosen": 0.00032176534296013415,
      "rewards/margins": -2.4562166800023988e-05,
      "rewards/rejected": 0.00034632752067409456,
      "step": 300
    },
    {
      "epoch": 0.96,
      "eval_logits/chosen": -0.8510691523551941,
      "eval_logits/rejected": -0.7548762559890747,
      "eval_logps/chosen": -394.70794677734375,
      "eval_logps/rejected": -365.4079284667969,
      "eval_loss": 0.053816698491573334,
      "eval_rewards/accuracies": 0.4410000145435333,
      "eval_rewards/chosen": 0.0015973460394889116,
      "eval_rewards/margins": -0.0002301189670106396,
      "eval_rewards/rejected": 0.0018274650210514665,
      "eval_runtime": 545.8177,
      "eval_samples_per_second": 3.664,
      "eval_steps_per_second": 0.916,
      "step": 300
    },
    {
      "epoch": 0.99,
      "learning_rate": 6.294126437336734e-10,
      "logits/chosen": -1.0546523332595825,
      "logits/rejected": -0.9914261698722839,
      "logps/chosen": -20.59957504272461,
      "logps/rejected": -15.304788589477539,
      "loss": 0.1463,
      "rewards/accuracies": 0.02500000037252903,
      "rewards/chosen": 0.0003310924512334168,
      "rewards/margins": 6.335419311653823e-05,
      "rewards/rejected": 0.00026773830177262425,
      "step": 310
    },
    {
      "epoch": 1.0,
      "step": 312,
      "total_flos": 0.0,
      "train_loss": 0.14698060888510484,
      "train_runtime": 3810.9697,
      "train_samples_per_second": 1.312,
      "train_steps_per_second": 0.082
    }
  ],
  "logging_steps": 10,
  "max_steps": 312,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}