{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984591679506933,
  "eval_steps": 100,
  "global_step": 324,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 39.87631410320537,
      "learning_rate": 1.5151515151515152e-08,
      "logits/chosen": -3.1684141159057617,
      "logits/rejected": -3.1765036582946777,
      "logps/chosen": -1262.7908935546875,
      "logps/rejected": -1304.270263671875,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "grad_norm": 44.090928533988176,
      "learning_rate": 1.5151515151515152e-07,
      "logits/chosen": -3.145017385482788,
      "logits/rejected": -3.17344069480896,
      "logps/chosen": -1035.2520751953125,
      "logps/rejected": -1331.3636474609375,
      "loss": 0.6917,
      "rewards/accuracies": 0.5277777910232544,
      "rewards/chosen": 0.003002108307555318,
      "rewards/margins": 0.0036764023825526237,
      "rewards/rejected": -0.0006742942496202886,
      "step": 10
    },
    {
      "epoch": 0.06,
      "grad_norm": 32.37744800941447,
      "learning_rate": 3.0303030303030305e-07,
      "logits/chosen": -3.1358423233032227,
      "logits/rejected": -3.18705415725708,
      "logps/chosen": -968.2097778320312,
      "logps/rejected": -1354.069580078125,
      "loss": 0.6617,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.03913033753633499,
      "rewards/margins": 0.06312780827283859,
      "rewards/rejected": -0.023997480049729347,
      "step": 20
    },
    {
      "epoch": 0.09,
      "grad_norm": 30.93962012271263,
      "learning_rate": 4.545454545454545e-07,
      "logits/chosen": -3.2511069774627686,
      "logits/rejected": -3.244719982147217,
      "logps/chosen": -1036.7672119140625,
      "logps/rejected": -1373.1820068359375,
      "loss": 0.5896,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.06491168588399887,
      "rewards/margins": 0.31840670108795166,
      "rewards/rejected": -0.2534949779510498,
      "step": 30
    },
    {
      "epoch": 0.12,
      "grad_norm": 31.51946350483435,
      "learning_rate": 4.992864684782648e-07,
      "logits/chosen": -3.3422675132751465,
      "logits/rejected": -3.370623826980591,
      "logps/chosen": -1083.17431640625,
      "logps/rejected": -1492.5845947265625,
      "loss": 0.5125,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07627250999212265,
      "rewards/margins": 0.8225336074829102,
      "rewards/rejected": -0.898806095123291,
      "step": 40
    },
    {
      "epoch": 0.15,
      "grad_norm": 29.782679812110892,
      "learning_rate": 4.958014217656854e-07,
      "logits/chosen": -3.3696506023406982,
      "logits/rejected": -3.4038467407226562,
      "logps/chosen": -1081.7869873046875,
      "logps/rejected": -1461.259033203125,
      "loss": 0.4171,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.1066322773694992,
      "rewards/margins": 1.0666204690933228,
      "rewards/rejected": -1.173252820968628,
      "step": 50
    },
    {
      "epoch": 0.18,
      "grad_norm": 30.594547647279217,
      "learning_rate": 4.894543310469967e-07,
      "logits/chosen": -3.352465867996216,
      "logits/rejected": -3.3652706146240234,
      "logps/chosen": -1111.260009765625,
      "logps/rejected": -1504.715087890625,
      "loss": 0.3969,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.2318076640367508,
      "rewards/margins": 1.2468664646148682,
      "rewards/rejected": -1.4786741733551025,
      "step": 60
    },
    {
      "epoch": 0.22,
      "grad_norm": 31.824817232007625,
      "learning_rate": 4.803191000971128e-07,
      "logits/chosen": -3.3312506675720215,
      "logits/rejected": -3.355130434036255,
      "logps/chosen": -968.1290893554688,
      "logps/rejected": -1600.333251953125,
      "loss": 0.3874,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.22658078372478485,
      "rewards/margins": 2.3517754077911377,
      "rewards/rejected": -2.5783562660217285,
      "step": 70
    },
    {
      "epoch": 0.25,
      "grad_norm": 25.823223858100576,
      "learning_rate": 4.685020970273189e-07,
      "logits/chosen": -3.2700467109680176,
      "logits/rejected": -3.3080413341522217,
      "logps/chosen": -1002.4366455078125,
      "logps/rejected": -1593.41796875,
      "loss": 0.3546,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.07068847864866257,
      "rewards/margins": 2.2586522102355957,
      "rewards/rejected": -2.329341173171997,
      "step": 80
    },
    {
      "epoch": 0.28,
      "grad_norm": 43.928369861559965,
      "learning_rate": 4.541409157643027e-07,
      "logits/chosen": -3.235419511795044,
      "logits/rejected": -3.2496044635772705,
      "logps/chosen": -956.4049072265625,
      "logps/rejected": -1599.0389404296875,
      "loss": 0.3143,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 0.038097791373729706,
      "rewards/margins": 2.2678751945495605,
      "rewards/rejected": -2.2297775745391846,
      "step": 90
    },
    {
      "epoch": 0.31,
      "grad_norm": 30.820532733997354,
      "learning_rate": 4.374027739443952e-07,
      "logits/chosen": -3.204524517059326,
      "logits/rejected": -3.163343906402588,
      "logps/chosen": -1068.4237060546875,
      "logps/rejected": -1704.1986083984375,
      "loss": 0.2799,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.3904297947883606,
      "rewards/margins": 3.4663283824920654,
      "rewards/rejected": -3.8567581176757812,
      "step": 100
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": -3.0348000526428223,
      "eval_logits/rejected": -3.0867843627929688,
      "eval_logps/chosen": -584.1478881835938,
      "eval_logps/rejected": -794.01025390625,
      "eval_loss": 0.5261008143424988,
      "eval_rewards/accuracies": 0.75,
      "eval_rewards/chosen": -1.5201810598373413,
      "eval_rewards/margins": 0.9906590580940247,
      "eval_rewards/rejected": -2.5108399391174316,
      "eval_runtime": 34.7053,
      "eval_samples_per_second": 7.261,
      "eval_steps_per_second": 0.231,
      "step": 100
    },
    {
      "epoch": 0.34,
      "grad_norm": 47.845448282397456,
      "learning_rate": 4.184825658775027e-07,
      "logits/chosen": -3.128324031829834,
      "logits/rejected": -3.134152889251709,
      "logps/chosen": -1042.473388671875,
      "logps/rejected": -1787.997802734375,
      "loss": 0.2816,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.6833322644233704,
      "rewards/margins": 3.4197134971618652,
      "rewards/rejected": -4.10304594039917,
      "step": 110
    },
    {
      "epoch": 0.37,
      "grad_norm": 24.2664669682948,
      "learning_rate": 3.9760059325148063e-07,
      "logits/chosen": -3.1436760425567627,
      "logits/rejected": -3.091614246368408,
      "logps/chosen": -1067.834716796875,
      "logps/rejected": -1788.0120849609375,
      "loss": 0.2536,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.610935389995575,
      "rewards/margins": 4.445748329162598,
      "rewards/rejected": -5.056683540344238,
      "step": 120
    },
    {
      "epoch": 0.4,
      "grad_norm": 22.23462347593175,
      "learning_rate": 3.75e-07,
      "logits/chosen": -3.1414103507995605,
      "logits/rejected": -3.0941264629364014,
      "logps/chosen": -1100.4937744140625,
      "logps/rejected": -1801.8560791015625,
      "loss": 0.2298,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.540351152420044,
      "rewards/margins": 3.7757785320281982,
      "rewards/rejected": -4.3161301612854,
      "step": 130
    },
    {
      "epoch": 0.43,
      "grad_norm": 28.613362043744857,
      "learning_rate": 3.509439412016004e-07,
      "logits/chosen": -3.0641441345214844,
      "logits/rejected": -3.0451717376708984,
      "logps/chosen": -1098.5340576171875,
      "logps/rejected": -1918.6890869140625,
      "loss": 0.2135,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.7187612056732178,
      "rewards/margins": 5.323573112487793,
      "rewards/rejected": -6.04233455657959,
      "step": 140
    },
    {
      "epoch": 0.46,
      "grad_norm": 30.54616548038225,
      "learning_rate": 3.2571251897448763e-07,
      "logits/chosen": -2.992375135421753,
      "logits/rejected": -2.95180606842041,
      "logps/chosen": -1197.9376220703125,
      "logps/rejected": -2077.058349609375,
      "loss": 0.1801,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.5255634784698486,
      "rewards/margins": 5.869881629943848,
      "rewards/rejected": -7.395445346832275,
      "step": 150
    },
    {
      "epoch": 0.49,
      "grad_norm": 36.74589910484145,
      "learning_rate": 2.9959952104467243e-07,
      "logits/chosen": -2.9339356422424316,
      "logits/rejected": -2.85386323928833,
      "logps/chosen": -1247.737060546875,
      "logps/rejected": -2310.10205078125,
      "loss": 0.1778,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.0261478424072266,
      "rewards/margins": 7.8707451820373535,
      "rewards/rejected": -9.896891593933105,
      "step": 160
    },
    {
      "epoch": 0.52,
      "grad_norm": 32.132075393104884,
      "learning_rate": 2.729089999626637e-07,
      "logits/chosen": -2.980856418609619,
      "logits/rejected": -2.856822967529297,
      "logps/chosen": -1185.372802734375,
      "logps/rejected": -2347.78076171875,
      "loss": 0.1698,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.51887047290802,
      "rewards/margins": 8.371113777160645,
      "rewards/rejected": -9.889985084533691,
      "step": 170
    },
    {
      "epoch": 0.55,
      "grad_norm": 26.839790210789428,
      "learning_rate": 2.459517327993746e-07,
      "logits/chosen": -2.962564468383789,
      "logits/rejected": -2.8451316356658936,
      "logps/chosen": -1266.3397216796875,
      "logps/rejected": -2263.588623046875,
      "loss": 0.141,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7179988622665405,
      "rewards/margins": 7.524572849273682,
      "rewards/rejected": -9.242570877075195,
      "step": 180
    },
    {
      "epoch": 0.59,
      "grad_norm": 33.374896332465084,
      "learning_rate": 2.1904160254356748e-07,
      "logits/chosen": -2.881953477859497,
      "logits/rejected": -2.7538435459136963,
      "logps/chosen": -1169.200927734375,
      "logps/rejected": -2476.24072265625,
      "loss": 0.1207,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6462358236312866,
      "rewards/margins": 9.677408218383789,
      "rewards/rejected": -11.323644638061523,
      "step": 190
    },
    {
      "epoch": 0.62,
      "grad_norm": 44.90567050679125,
      "learning_rate": 1.9249194333484563e-07,
      "logits/chosen": -2.8342463970184326,
      "logits/rejected": -2.730264902114868,
      "logps/chosen": -1229.9298095703125,
      "logps/rejected": -2291.61181640625,
      "loss": 0.154,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.0603644847869873,
      "rewards/margins": 7.22509765625,
      "rewards/rejected": -9.285462379455566,
      "step": 200
    },
    {
      "epoch": 0.62,
      "eval_logits/chosen": -2.694772720336914,
      "eval_logits/rejected": -2.554710626602173,
      "eval_logps/chosen": -742.1358642578125,
      "eval_logps/rejected": -1446.8753662109375,
      "eval_loss": 0.09226308017969131,
      "eval_rewards/accuracies": 0.9375,
      "eval_rewards/chosen": -3.100059986114502,
      "eval_rewards/margins": 5.939432144165039,
      "eval_rewards/rejected": -9.0394926071167,
      "eval_runtime": 34.5072,
      "eval_samples_per_second": 7.303,
      "eval_steps_per_second": 0.232,
      "step": 200
    },
    {
      "epoch": 0.65,
      "grad_norm": 29.94298857126716,
      "learning_rate": 1.6661189208729489e-07,
      "logits/chosen": -2.786771059036255,
      "logits/rejected": -2.6327857971191406,
      "logps/chosen": -1283.205810546875,
      "logps/rejected": -2401.399169921875,
      "loss": 0.1526,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.189497709274292,
      "rewards/margins": 8.632684707641602,
      "rewards/rejected": -10.822182655334473,
      "step": 210
    },
    {
      "epoch": 0.68,
      "grad_norm": 33.13137270748439,
      "learning_rate": 1.4170278898446175e-07,
      "logits/chosen": -2.828369379043579,
      "logits/rejected": -2.650123119354248,
      "logps/chosen": -1175.46826171875,
      "logps/rejected": -2410.84326171875,
      "loss": 0.1252,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.8023840188980103,
      "rewards/margins": 8.576199531555176,
      "rewards/rejected": -10.378583908081055,
      "step": 220
    },
    {
      "epoch": 0.71,
      "grad_norm": 30.216897410019698,
      "learning_rate": 1.1805466875731276e-07,
      "logits/chosen": -2.820298671722412,
      "logits/rejected": -2.622697591781616,
      "logps/chosen": -1135.1295166015625,
      "logps/rejected": -2719.731201171875,
      "loss": 0.1353,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7641971111297607,
      "rewards/margins": 11.304890632629395,
      "rewards/rejected": -13.06908893585205,
      "step": 230
    },
    {
      "epoch": 0.74,
      "grad_norm": 28.800219923929006,
      "learning_rate": 9.594288359976815e-08,
      "logits/chosen": -2.815680742263794,
      "logits/rejected": -2.6530845165252686,
      "logps/chosen": -1304.4205322265625,
      "logps/rejected": -2253.841064453125,
      "loss": 0.1093,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9788525104522705,
      "rewards/margins": 7.066276550292969,
      "rewards/rejected": -9.045129776000977,
      "step": 240
    },
    {
      "epoch": 0.77,
      "grad_norm": 28.53520064286844,
      "learning_rate": 7.56248970436493e-08,
      "logits/chosen": -2.7308030128479004,
      "logits/rejected": -2.56375789642334,
      "logps/chosen": -1186.9593505859375,
      "logps/rejected": -2636.3701171875,
      "loss": 0.0913,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.045689105987549,
      "rewards/margins": 10.186556816101074,
      "rewards/rejected": -12.232245445251465,
      "step": 250
    },
    {
      "epoch": 0.8,
      "grad_norm": 21.7882636792373,
      "learning_rate": 5.733728612427771e-08,
      "logits/chosen": -2.7645225524902344,
      "logits/rejected": -2.5233638286590576,
      "logps/chosen": -1321.7745361328125,
      "logps/rejected": -2658.189453125,
      "loss": 0.1155,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.5851030349731445,
      "rewards/margins": 10.245210647583008,
      "rewards/rejected": -12.830312728881836,
      "step": 260
    },
    {
      "epoch": 0.83,
      "grad_norm": 34.32906854711248,
      "learning_rate": 4.1292986742682254e-08,
      "logits/chosen": -2.668457508087158,
      "logits/rejected": -2.500288963317871,
      "logps/chosen": -1262.6650390625,
      "logps/rejected": -2782.10009765625,
      "loss": 0.0989,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.7102155685424805,
      "rewards/margins": 11.686820983886719,
      "rewards/rejected": -14.3970365524292,
      "step": 270
    },
    {
      "epoch": 0.86,
      "grad_norm": 42.763438453906815,
      "learning_rate": 2.7678814298657732e-08,
      "logits/chosen": -2.6972427368164062,
      "logits/rejected": -2.4791617393493652,
      "logps/chosen": -1353.071533203125,
      "logps/rejected": -2790.54052734375,
      "loss": 0.1022,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.827423095703125,
      "rewards/margins": 11.204734802246094,
      "rewards/rejected": -14.032157897949219,
      "step": 280
    },
    {
      "epoch": 0.89,
      "grad_norm": 26.418126829556318,
      "learning_rate": 1.6653288463741062e-08,
      "logits/chosen": -2.689786672592163,
      "logits/rejected": -2.518730401992798,
      "logps/chosen": -1242.576416015625,
      "logps/rejected": -2554.541748046875,
      "loss": 0.1067,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.544224262237549,
      "rewards/margins": 9.17101001739502,
      "rewards/rejected": -11.715234756469727,
      "step": 290
    },
    {
      "epoch": 0.92,
      "grad_norm": 40.4422866987403,
      "learning_rate": 8.344787421847216e-09,
      "logits/chosen": -2.65974497795105,
      "logits/rejected": -2.4722535610198975,
      "logps/chosen": -1281.4610595703125,
      "logps/rejected": -2672.197265625,
      "loss": 0.0948,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.5506279468536377,
      "rewards/margins": 10.449880599975586,
      "rewards/rejected": -13.000508308410645,
      "step": 300
    },
    {
      "epoch": 0.92,
      "eval_logits/chosen": -2.5877139568328857,
      "eval_logits/rejected": -2.3930397033691406,
      "eval_logps/chosen": -803.1033325195312,
      "eval_logps/rejected": -1661.4266357421875,
      "eval_loss": 0.07533077150583267,
      "eval_rewards/accuracies": 0.953125,
      "eval_rewards/chosen": -3.70973539352417,
      "eval_rewards/margins": 7.4752678871154785,
      "eval_rewards/rejected": -11.185002326965332,
      "eval_runtime": 34.126,
      "eval_samples_per_second": 7.384,
      "eval_steps_per_second": 0.234,
      "step": 300
    },
    {
      "epoch": 0.96,
      "grad_norm": 38.70439473653127,
      "learning_rate": 2.850053069080344e-09,
      "logits/chosen": -2.730034351348877,
      "logits/rejected": -2.4978787899017334,
      "logps/chosen": -1227.954345703125,
      "logps/rejected": -2714.13623046875,
      "loss": 0.1114,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.477766990661621,
      "rewards/margins": 11.021059036254883,
      "rewards/rejected": -13.498825073242188,
      "step": 310
    },
    {
      "epoch": 0.99,
      "grad_norm": 25.381541507877866,
      "learning_rate": 2.3306457775981727e-10,
      "logits/chosen": -2.6724932193756104,
      "logits/rejected": -2.4461209774017334,
      "logps/chosen": -1288.38623046875,
      "logps/rejected": -2817.382568359375,
      "loss": 0.0963,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.566399097442627,
      "rewards/margins": 11.983835220336914,
      "rewards/rejected": -14.550233840942383,
      "step": 320
    },
    {
      "epoch": 1.0,
      "step": 324,
      "total_flos": 0.0,
      "train_loss": 0.007740737387427577,
      "train_runtime": 396.513,
      "train_samples_per_second": 52.306,
      "train_steps_per_second": 0.817
    }
  ],
  "logging_steps": 10,
  "max_steps": 324,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}