{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 309,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 6.266341746346053,
      "learning_rate": 1.6129032258064518e-07,
      "logits/chosen": -0.5416143536567688,
      "logits/rejected": -0.9699263572692871,
      "logps/chosen": -998.3239135742188,
      "logps/rejected": -1286.9267578125,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "grad_norm": 6.79176132535156,
      "learning_rate": 1.6129032258064516e-06,
      "logits/chosen": -0.501338541507721,
      "logits/rejected": -0.6205970048904419,
      "logps/chosen": -965.1869506835938,
      "logps/rejected": -1388.869140625,
      "loss": 0.6908,
      "rewards/accuracies": 0.4791666567325592,
      "rewards/chosen": -0.005239578895270824,
      "rewards/margins": 0.0034307329915463924,
      "rewards/rejected": -0.00867031142115593,
      "step": 10
    },
    {
      "epoch": 0.06,
      "grad_norm": 5.46084581758702,
      "learning_rate": 3.225806451612903e-06,
      "logits/chosen": -0.4920225143432617,
      "logits/rejected": -0.5186491012573242,
      "logps/chosen": -1001.7984619140625,
      "logps/rejected": -1416.242431640625,
      "loss": 0.6367,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.13937075436115265,
      "rewards/margins": 0.18681207299232483,
      "rewards/rejected": -0.32618284225463867,
      "step": 20
    },
    {
      "epoch": 0.1,
      "grad_norm": 5.423152706060893,
      "learning_rate": 4.838709677419355e-06,
      "logits/chosen": -0.3083574175834656,
      "logits/rejected": -0.28873246908187866,
      "logps/chosen": -936.2586669921875,
      "logps/rejected": -1426.4195556640625,
      "loss": 0.4832,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.22132034599781036,
      "rewards/margins": 0.7907422780990601,
      "rewards/rejected": -1.012062668800354,
      "step": 30
    },
    {
      "epoch": 0.13,
      "grad_norm": 4.497512811390226,
      "learning_rate": 4.987080943856887e-06,
      "logits/chosen": -0.39709392189979553,
      "logits/rejected": -0.3410794734954834,
      "logps/chosen": -864.56787109375,
      "logps/rejected": -1415.4027099609375,
      "loss": 0.431,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.10076986253261566,
      "rewards/margins": 1.5246957540512085,
      "rewards/rejected": -1.625465750694275,
      "step": 40
    },
    {
      "epoch": 0.16,
      "grad_norm": 5.38009380317694,
      "learning_rate": 4.942593872763566e-06,
      "logits/chosen": -0.2893625795841217,
      "logits/rejected": -0.3507555425167084,
      "logps/chosen": -904.3942260742188,
      "logps/rejected": -1616.10986328125,
      "loss": 0.3741,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.2879861891269684,
      "rewards/margins": 2.9569807052612305,
      "rewards/rejected": -3.244966983795166,
      "step": 50
    },
    {
      "epoch": 0.19,
      "grad_norm": 4.021367545314158,
      "learning_rate": 4.866946677079314e-06,
      "logits/chosen": -0.2734339237213135,
      "logits/rejected": -0.2639048397541046,
      "logps/chosen": -1028.2137451171875,
      "logps/rejected": -1476.271728515625,
      "loss": 0.349,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.4066247045993805,
      "rewards/margins": 1.8377554416656494,
      "rewards/rejected": -2.244380235671997,
      "step": 60
    },
    {
      "epoch": 0.23,
      "grad_norm": 5.598262404371225,
      "learning_rate": 4.761104386672074e-06,
      "logits/chosen": -0.1697017401456833,
      "logits/rejected": -0.25459176301956177,
      "logps/chosen": -949.3585205078125,
      "logps/rejected": -1626.849365234375,
      "loss": 0.3077,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.5060699582099915,
      "rewards/margins": 2.161994457244873,
      "rewards/rejected": -2.6680643558502197,
      "step": 70
    },
    {
      "epoch": 0.26,
      "grad_norm": 4.260536872127374,
      "learning_rate": 4.626417229671401e-06,
      "logits/chosen": -0.22611574828624725,
      "logits/rejected": -0.2613917291164398,
      "logps/chosen": -920.0872192382812,
      "logps/rejected": -1731.564453125,
      "loss": 0.2782,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.45848578214645386,
      "rewards/margins": 3.2202796936035156,
      "rewards/rejected": -3.6787655353546143,
      "step": 80
    },
    {
      "epoch": 0.29,
      "grad_norm": 5.248398944622536,
      "learning_rate": 4.464603407633326e-06,
      "logits/chosen": -0.20690715312957764,
      "logits/rejected": -0.3079308271408081,
      "logps/chosen": -1012.1229248046875,
      "logps/rejected": -1682.8330078125,
      "loss": 0.2599,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.40270423889160156,
      "rewards/margins": 3.2318668365478516,
      "rewards/rejected": -3.634571075439453,
      "step": 90
    },
    {
      "epoch": 0.32,
      "grad_norm": 4.915340212981017,
      "learning_rate": 4.2777271764750805e-06,
      "logits/chosen": -0.27068111300468445,
      "logits/rejected": -0.22187161445617676,
      "logps/chosen": -952.927734375,
      "logps/rejected": -1708.844482421875,
      "loss": 0.2249,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.5603285431861877,
      "rewards/margins": 3.670332670211792,
      "rewards/rejected": -4.230660915374756,
      "step": 100
    },
    {
      "epoch": 0.32,
      "eval_logits/chosen": -0.1106632873415947,
      "eval_logits/rejected": -0.028987836092710495,
      "eval_logps/chosen": -650.2339477539062,
      "eval_logps/rejected": -1190.2701416015625,
      "eval_loss": 0.3820802569389343,
      "eval_rewards/accuracies": 0.8551136255264282,
      "eval_rewards/chosen": -0.9338886141777039,
      "eval_rewards/margins": 3.643230438232422,
      "eval_rewards/rejected": -4.577118873596191,
      "eval_runtime": 178.7591,
      "eval_samples_per_second": 7.787,
      "eval_steps_per_second": 0.246,
      "step": 100
    },
    {
      "epoch": 0.36,
      "grad_norm": 3.910365611216077,
      "learning_rate": 4.06817251280076e-06,
      "logits/chosen": -0.20240063965320587,
      "logits/rejected": -0.265985906124115,
      "logps/chosen": -1035.5126953125,
      "logps/rejected": -1685.556640625,
      "loss": 0.2552,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.44126248359680176,
      "rewards/margins": 3.790078639984131,
      "rewards/rejected": -4.231341361999512,
      "step": 110
    },
    {
      "epoch": 0.39,
      "grad_norm": 9.625288099258173,
      "learning_rate": 3.838612701556138e-06,
      "logits/chosen": -0.243825763463974,
      "logits/rejected": -0.3290537893772125,
      "logps/chosen": -967.8455810546875,
      "logps/rejected": -1898.212646484375,
      "loss": 0.2415,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.19587793946266174,
      "rewards/margins": 3.995615005493164,
      "rewards/rejected": -4.191493034362793,
      "step": 120
    },
    {
      "epoch": 0.42,
      "grad_norm": 3.4423605201536476,
      "learning_rate": 3.5919762329823556e-06,
      "logits/chosen": -0.22354824841022491,
      "logits/rejected": -0.2977936565876007,
      "logps/chosen": -962.7554931640625,
      "logps/rejected": -1781.5015869140625,
      "loss": 0.2134,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.3282354772090912,
      "rewards/margins": 4.096698760986328,
      "rewards/rejected": -4.424933910369873,
      "step": 130
    },
    {
      "epoch": 0.45,
      "grad_norm": 4.355526701031263,
      "learning_rate": 3.3314094439203903e-06,
      "logits/chosen": -0.21654577553272247,
      "logits/rejected": -0.3782255947589874,
      "logps/chosen": -956.0657348632812,
      "logps/rejected": -1783.807861328125,
      "loss": 0.2054,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.5269081592559814,
      "rewards/margins": 4.415614128112793,
      "rewards/rejected": -4.9425225257873535,
      "step": 140
    },
    {
      "epoch": 0.49,
      "grad_norm": 4.577252564770749,
      "learning_rate": 3.0602363800505198e-06,
      "logits/chosen": -0.21595144271850586,
      "logits/rejected": -0.3744010925292969,
      "logps/chosen": -1055.1593017578125,
      "logps/rejected": -1893.0072021484375,
      "loss": 0.1989,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3978404402732849,
      "rewards/margins": 5.1758904457092285,
      "rewards/rejected": -5.573730945587158,
      "step": 150
    },
    {
      "epoch": 0.52,
      "grad_norm": 4.231494324857874,
      "learning_rate": 2.7819163911034175e-06,
      "logits/chosen": -0.22502343356609344,
      "logits/rejected": -0.37728679180145264,
      "logps/chosen": -989.0603637695312,
      "logps/rejected": -1922.759521484375,
      "loss": 0.1431,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.17479470372200012,
      "rewards/margins": 5.446383476257324,
      "rewards/rejected": -5.62117862701416,
      "step": 160
    },
    {
      "epoch": 0.55,
      "grad_norm": 6.2160398163872665,
      "learning_rate": 2.5e-06,
      "logits/chosen": -0.2915242314338684,
      "logits/rejected": -0.45713406801223755,
      "logps/chosen": -1042.622314453125,
      "logps/rejected": -2141.868896484375,
      "loss": 0.1686,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.609991192817688,
      "rewards/margins": 7.452821254730225,
      "rewards/rejected": -8.062813758850098,
      "step": 170
    },
    {
      "epoch": 0.58,
      "grad_norm": 2.781036420408215,
      "learning_rate": 2.2180836088965833e-06,
      "logits/chosen": -0.3469601273536682,
      "logits/rejected": -0.5014016032218933,
      "logps/chosen": -1045.5155029296875,
      "logps/rejected": -1821.9827880859375,
      "loss": 0.1551,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07482553273439407,
      "rewards/margins": 5.159177303314209,
      "rewards/rejected": -5.234003067016602,
      "step": 180
    },
    {
      "epoch": 0.61,
      "grad_norm": 3.3125651058366268,
      "learning_rate": 1.939763619949481e-06,
      "logits/chosen": -0.35754817724227905,
      "logits/rejected": -0.5549635887145996,
      "logps/chosen": -864.7000732421875,
      "logps/rejected": -1795.1302490234375,
      "loss": 0.1376,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.1424350142478943,
      "rewards/margins": 4.835862636566162,
      "rewards/rejected": -4.978297710418701,
      "step": 190
    },
    {
      "epoch": 0.65,
      "grad_norm": 4.010712916190072,
      "learning_rate": 1.6685905560796101e-06,
      "logits/chosen": -0.46689772605895996,
      "logits/rejected": -0.6180375814437866,
      "logps/chosen": -1065.89599609375,
      "logps/rejected": -2047.24609375,
      "loss": 0.1549,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5212541222572327,
      "rewards/margins": 6.1373467445373535,
      "rewards/rejected": -6.658600807189941,
      "step": 200
    },
    {
      "epoch": 0.65,
      "eval_logits/chosen": -0.3179807960987091,
      "eval_logits/rejected": -0.32223665714263916,
      "eval_logps/chosen": -652.9113159179688,
      "eval_logps/rejected": -1308.40478515625,
      "eval_loss": 0.27093741297721863,
      "eval_rewards/accuracies": 0.8977272510528564,
      "eval_rewards/chosen": -0.960662305355072,
      "eval_rewards/margins": 4.797802925109863,
      "eval_rewards/rejected": -5.758464813232422,
      "eval_runtime": 173.283,
      "eval_samples_per_second": 8.033,
      "eval_steps_per_second": 0.254,
      "step": 200
    },
    {
      "epoch": 0.68,
      "grad_norm": 3.8323612831804175,
      "learning_rate": 1.4080237670176456e-06,
      "logits/chosen": -0.4167974889278412,
      "logits/rejected": -0.5508753061294556,
      "logps/chosen": -1013.05419921875,
      "logps/rejected": -1894.520263671875,
      "loss": 0.1443,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.12083841860294342,
      "rewards/margins": 5.549345970153809,
      "rewards/rejected": -5.67018461227417,
      "step": 210
    },
    {
      "epoch": 0.71,
      "grad_norm": 4.939784171225143,
      "learning_rate": 1.161387298443863e-06,
      "logits/chosen": -0.3491138815879822,
      "logits/rejected": -0.5552398562431335,
      "logps/chosen": -902.9434814453125,
      "logps/rejected": -1838.8209228515625,
      "loss": 0.148,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.21528509259223938,
      "rewards/margins": 5.609208583831787,
      "rewards/rejected": -5.824493408203125,
      "step": 220
    },
    {
      "epoch": 0.74,
      "grad_norm": 5.989265642846245,
      "learning_rate": 9.318274871992408e-07,
      "logits/chosen": -0.44859474897384644,
      "logits/rejected": -0.6015830636024475,
      "logps/chosen": -1067.4637451171875,
      "logps/rejected": -2109.045654296875,
      "loss": 0.157,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.305181622505188,
      "rewards/margins": 7.274392604827881,
      "rewards/rejected": -7.579575538635254,
      "step": 230
    },
    {
      "epoch": 0.78,
      "grad_norm": 4.078696847368483,
      "learning_rate": 7.222728235249196e-07,
      "logits/chosen": -0.4398832321166992,
      "logits/rejected": -0.5384049415588379,
      "logps/chosen": -971.4193115234375,
      "logps/rejected": -1864.9664306640625,
      "loss": 0.1321,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.1642368584871292,
      "rewards/margins": 5.839243412017822,
      "rewards/rejected": -6.003479957580566,
      "step": 240
    },
    {
      "epoch": 0.81,
      "grad_norm": 2.772016531721353,
      "learning_rate": 5.353965923666743e-07,
      "logits/chosen": -0.4434467852115631,
      "logits/rejected": -0.5806728601455688,
      "logps/chosen": -985.8177490234375,
      "logps/rejected": -1902.5126953125,
      "loss": 0.125,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.21859340369701385,
      "rewards/margins": 5.029125213623047,
      "rewards/rejected": -5.247718334197998,
      "step": 250
    },
    {
      "epoch": 0.84,
      "grad_norm": 3.5375826568350557,
      "learning_rate": 3.7358277032860016e-07,
      "logits/chosen": -0.47191086411476135,
      "logits/rejected": -0.5328727960586548,
      "logps/chosen": -913.7701416015625,
      "logps/rejected": -2043.599365234375,
      "loss": 0.1244,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4412022531032562,
      "rewards/margins": 6.3683366775512695,
      "rewards/rejected": -6.809537410736084,
      "step": 260
    },
    {
      "epoch": 0.87,
      "grad_norm": 3.1823409653630907,
      "learning_rate": 2.388956133279266e-07,
      "logits/chosen": -0.3901548683643341,
      "logits/rejected": -0.4868335723876953,
      "logps/chosen": -891.0558471679688,
      "logps/rejected": -1852.1605224609375,
      "loss": 0.1274,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5433205366134644,
      "rewards/margins": 5.383849620819092,
      "rewards/rejected": -5.927170753479004,
      "step": 270
    },
    {
      "epoch": 0.91,
      "grad_norm": 2.5807533458880316,
      "learning_rate": 1.3305332292068706e-07,
      "logits/chosen": -0.3871026039123535,
      "logits/rejected": -0.5441304445266724,
      "logps/chosen": -887.1787109375,
      "logps/rejected": -1961.8681640625,
      "loss": 0.0871,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.31611892580986023,
      "rewards/margins": 5.871817588806152,
      "rewards/rejected": -6.187936782836914,
      "step": 280
    },
    {
      "epoch": 0.94,
      "grad_norm": 2.1667077758960303,
      "learning_rate": 5.7406127236434016e-08,
      "logits/chosen": -0.45710650086402893,
      "logits/rejected": -0.6058652400970459,
      "logps/chosen": -984.42578125,
      "logps/rejected": -2196.794189453125,
      "loss": 0.0998,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6274508237838745,
      "rewards/margins": 7.608497619628906,
      "rewards/rejected": -8.23594856262207,
      "step": 290
    },
    {
      "epoch": 0.97,
      "grad_norm": 2.549184554041034,
      "learning_rate": 1.2919056143113062e-08,
      "logits/chosen": -0.4359508156776428,
      "logits/rejected": -0.6320601105690002,
      "logps/chosen": -949.6388549804688,
      "logps/rejected": -2248.93798828125,
      "loss": 0.0946,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.5125720500946045,
      "rewards/margins": 7.058285713195801,
      "rewards/rejected": -7.570857048034668,
      "step": 300
    },
    {
      "epoch": 0.97,
      "eval_logits/chosen": -0.3523492217063904,
      "eval_logits/rejected": -0.3282558023929596,
      "eval_logps/chosen": -679.615478515625,
      "eval_logps/rejected": -1442.4718017578125,
      "eval_loss": 0.27558115124702454,
      "eval_rewards/accuracies": 0.8920454382896423,
      "eval_rewards/chosen": -1.2277040481567383,
      "eval_rewards/margins": 5.87143087387085,
      "eval_rewards/rejected": -7.099134922027588,
      "eval_runtime": 171.6402,
      "eval_samples_per_second": 8.11,
      "eval_steps_per_second": 0.256,
      "step": 300
    },
    {
      "epoch": 1.0,
      "step": 309,
      "total_flos": 0.0,
      "train_loss": 0.0030385508506429234,
      "train_runtime": 144.6456,
      "train_samples_per_second": 136.617,
      "train_steps_per_second": 2.136
    }
  ],
  "logging_steps": 10,
  "max_steps": 309,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}