{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 328,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 52.10260886950557,
      "learning_rate": 1.5151515151515152e-08,
      "logits/chosen": -2.9049293994903564,
      "logits/rejected": -3.137899398803711,
      "logps/chosen": -848.0860595703125,
      "logps/rejected": -1267.928466796875,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "grad_norm": 54.32742967797558,
      "learning_rate": 1.5151515151515152e-07,
      "logits/chosen": -3.1098616123199463,
      "logits/rejected": -3.1124260425567627,
      "logps/chosen": -1029.3779296875,
      "logps/rejected": -1345.0582275390625,
      "loss": 0.6914,
      "rewards/accuracies": 0.4930555522441864,
      "rewards/chosen": 0.00401829881593585,
      "rewards/margins": 0.004102306440472603,
      "rewards/rejected": -8.400815568165854e-05,
      "step": 10
    },
    {
      "epoch": 0.06,
      "grad_norm": 31.172998615351542,
      "learning_rate": 3.0303030303030305e-07,
      "logits/chosen": -3.117793560028076,
      "logits/rejected": -3.1491177082061768,
      "logps/chosen": -1071.2353515625,
      "logps/rejected": -1354.4710693359375,
      "loss": 0.6537,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.07634462416172028,
      "rewards/margins": 0.08821593225002289,
      "rewards/rejected": -0.011871310882270336,
      "step": 20
    },
    {
      "epoch": 0.09,
      "grad_norm": 28.792134111609187,
      "learning_rate": 4.545454545454545e-07,
      "logits/chosen": -3.1649086475372314,
      "logits/rejected": -3.2152061462402344,
      "logps/chosen": -1064.19970703125,
      "logps/rejected": -1391.3843994140625,
      "loss": 0.5897,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.1755533069372177,
      "rewards/margins": 0.30122771859169006,
      "rewards/rejected": -0.12567442655563354,
      "step": 30
    },
    {
      "epoch": 0.12,
      "grad_norm": 26.037250152730635,
      "learning_rate": 4.993056783981012e-07,
      "logits/chosen": -3.338702440261841,
      "logits/rejected": -3.3210806846618652,
      "logps/chosen": -1098.9052734375,
      "logps/rejected": -1450.2767333984375,
      "loss": 0.4997,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 0.13215045630931854,
      "rewards/margins": 0.7883480787277222,
      "rewards/rejected": -0.6561976075172424,
      "step": 40
    },
    {
      "epoch": 0.15,
      "grad_norm": 27.996786342247532,
      "learning_rate": 4.95914200522199e-07,
      "logits/chosen": -3.2485861778259277,
      "logits/rejected": -3.324843168258667,
      "logps/chosen": -996.6271362304688,
      "logps/rejected": -1481.156982421875,
      "loss": 0.467,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.08611873537302017,
      "rewards/margins": 1.149458646774292,
      "rewards/rejected": -1.0633398294448853,
      "step": 50
    },
    {
      "epoch": 0.18,
      "grad_norm": 25.09282538012874,
      "learning_rate": 4.897364164920514e-07,
      "logits/chosen": -3.313831329345703,
      "logits/rejected": -3.2785778045654297,
      "logps/chosen": -959.1129150390625,
      "logps/rejected": -1587.156982421875,
      "loss": 0.3729,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 0.13842354714870453,
      "rewards/margins": 1.846784234046936,
      "rewards/rejected": -1.7083606719970703,
      "step": 60
    },
    {
      "epoch": 0.21,
      "grad_norm": 30.921162958697664,
      "learning_rate": 4.808423230692373e-07,
      "logits/chosen": -3.2474944591522217,
      "logits/rejected": -3.291019916534424,
      "logps/chosen": -1021.1806640625,
      "logps/rejected": -1590.413818359375,
      "loss": 0.3752,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.16332893073558807,
      "rewards/margins": 2.0271217823028564,
      "rewards/rejected": -2.190450429916382,
      "step": 70
    },
    {
      "epoch": 0.24,
      "grad_norm": 31.254868204859726,
      "learning_rate": 4.693326938861366e-07,
      "logits/chosen": -3.301940441131592,
      "logits/rejected": -3.3018264770507812,
      "logps/chosen": -1133.804443359375,
      "logps/rejected": -1672.321533203125,
      "loss": 0.3439,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7101578712463379,
      "rewards/margins": 2.033524751663208,
      "rewards/rejected": -2.743682861328125,
      "step": 80
    },
    {
      "epoch": 0.27,
      "grad_norm": 34.55073034392427,
      "learning_rate": 4.553379376404084e-07,
      "logits/chosen": -3.212740421295166,
      "logits/rejected": -3.263364791870117,
      "logps/chosen": -1189.436279296875,
      "logps/rejected": -1603.1971435546875,
      "loss": 0.3203,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.9354013204574585,
      "rewards/margins": 2.177839517593384,
      "rewards/rejected": -3.113241195678711,
      "step": 90
    },
    {
      "epoch": 0.3,
      "grad_norm": 44.68710091171008,
      "learning_rate": 4.3901662051233745e-07,
      "logits/chosen": -3.138298511505127,
      "logits/rejected": -3.148540496826172,
      "logps/chosen": -1065.9473876953125,
      "logps/rejected": -1705.128173828125,
      "loss": 0.2894,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.9562646150588989,
      "rewards/margins": 2.6660516262054443,
      "rewards/rejected": -3.622316360473633,
      "step": 100
    },
    {
      "epoch": 0.3,
      "eval_logits/chosen": -2.4093375205993652,
      "eval_logits/rejected": -2.694366216659546,
      "eval_logps/chosen": -111.92229461669922,
      "eval_logps/rejected": -801.1410522460938,
      "eval_loss": 0.2025093287229538,
      "eval_rewards/accuracies": 0.875,
      "eval_rewards/chosen": -0.21333497762680054,
      "eval_rewards/margins": 1.7935402393341064,
      "eval_rewards/rejected": -2.0068752765655518,
      "eval_runtime": 5.1839,
      "eval_samples_per_second": 4.63,
      "eval_steps_per_second": 0.193,
      "step": 100
    },
    {
      "epoch": 0.34,
      "grad_norm": 38.50657095083863,
      "learning_rate": 4.2055366954665237e-07,
      "logits/chosen": -3.1230928897857666,
      "logits/rejected": -3.1047117710113525,
      "logps/chosen": -1150.9886474609375,
      "logps/rejected": -1801.260009765625,
      "loss": 0.269,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.3049598932266235,
      "rewards/margins": 2.929929494857788,
      "rewards/rejected": -4.234889030456543,
      "step": 110
    },
    {
      "epoch": 0.37,
      "grad_norm": 29.424331589339463,
      "learning_rate": 4.0015827735521523e-07,
      "logits/chosen": -3.0130834579467773,
      "logits/rejected": -3.0237557888031006,
      "logps/chosen": -1210.0142822265625,
      "logps/rejected": -1803.6591796875,
      "loss": 0.2426,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.3954451084136963,
      "rewards/margins": 3.6785919666290283,
      "rewards/rejected": -5.074037551879883,
      "step": 120
    },
    {
      "epoch": 0.4,
      "grad_norm": 61.571819091215445,
      "learning_rate": 3.780615318811402e-07,
      "logits/chosen": -3.051529884338379,
      "logits/rejected": -2.9141249656677246,
      "logps/chosen": -1176.2293701171875,
      "logps/rejected": -2026.753173828125,
      "loss": 0.2321,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.6401822566986084,
      "rewards/margins": 4.810054779052734,
      "rewards/rejected": -6.450237274169922,
      "step": 130
    },
    {
      "epoch": 0.43,
      "grad_norm": 53.20520696391761,
      "learning_rate": 3.545137980800601e-07,
      "logits/chosen": -3.0094645023345947,
      "logits/rejected": -2.877675771713257,
      "logps/chosen": -1233.012451171875,
      "logps/rejected": -2089.841552734375,
      "loss": 0.1758,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.5989583730697632,
      "rewards/margins": 5.455811500549316,
      "rewards/rejected": -7.054769039154053,
      "step": 140
    },
    {
      "epoch": 0.46,
      "grad_norm": 39.12953948720105,
      "learning_rate": 3.2978188118513814e-07,
      "logits/chosen": -2.921603202819824,
      "logits/rejected": -2.7563977241516113,
      "logps/chosen": -1218.6588134765625,
      "logps/rejected": -2079.738037109375,
      "loss": 0.1843,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8553876876831055,
      "rewards/margins": 5.326550483703613,
      "rewards/rejected": -7.181939601898193,
      "step": 150
    },
    {
      "epoch": 0.49,
      "grad_norm": 51.519512868650644,
      "learning_rate": 3.041460036971664e-07,
      "logits/chosen": -2.8195321559906006,
      "logits/rejected": -2.6623053550720215,
      "logps/chosen": -1251.6900634765625,
      "logps/rejected": -2145.56591796875,
      "loss": 0.171,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.4222846031188965,
      "rewards/margins": 5.688368797302246,
      "rewards/rejected": -8.110652923583984,
      "step": 160
    },
    {
      "epoch": 0.52,
      "grad_norm": 34.8732410879268,
      "learning_rate": 2.778966303516603e-07,
      "logits/chosen": -2.7237300872802734,
      "logits/rejected": -2.5120363235473633,
      "logps/chosen": -1312.8253173828125,
      "logps/rejected": -2203.4736328125,
      "loss": 0.1537,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.7755582332611084,
      "rewards/margins": 5.876898288726807,
      "rewards/rejected": -8.652456283569336,
      "step": 170
    },
    {
      "epoch": 0.55,
      "grad_norm": 38.69272613867567,
      "learning_rate": 2.513311770373421e-07,
      "logits/chosen": -2.8833096027374268,
      "logits/rejected": -2.700117349624634,
      "logps/chosen": -1203.2437744140625,
      "logps/rejected": -2128.73876953125,
      "loss": 0.164,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8221490383148193,
      "rewards/margins": 5.741241455078125,
      "rewards/rejected": -7.563390254974365,
      "step": 180
    },
    {
      "epoch": 0.58,
      "grad_norm": 43.40638062311321,
      "learning_rate": 2.2475064095527946e-07,
      "logits/chosen": -2.8215672969818115,
      "logits/rejected": -2.522494316101074,
      "logps/chosen": -1261.7806396484375,
      "logps/rejected": -2357.189453125,
      "loss": 0.1105,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.5550084114074707,
      "rewards/margins": 7.024233341217041,
      "rewards/rejected": -9.579240798950195,
      "step": 190
    },
    {
      "epoch": 0.61,
      "grad_norm": 46.47610329796461,
      "learning_rate": 1.984561902003255e-07,
      "logits/chosen": -2.600424289703369,
      "logits/rejected": -2.3879284858703613,
      "logps/chosen": -1387.474365234375,
      "logps/rejected": -2327.524169921875,
      "loss": 0.1355,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.2148842811584473,
      "rewards/margins": 6.671014308929443,
      "rewards/rejected": -9.885897636413574,
      "step": 200
    },
    {
      "epoch": 0.61,
      "eval_logits/chosen": -1.9005341529846191,
      "eval_logits/rejected": -1.5654257535934448,
      "eval_logps/chosen": -174.2285614013672,
      "eval_logps/rejected": -1813.7420654296875,
      "eval_loss": 0.05062888190150261,
      "eval_rewards/accuracies": 0.875,
      "eval_rewards/chosen": -0.8363977074623108,
      "eval_rewards/margins": 11.296485900878906,
      "eval_rewards/rejected": -12.13288402557373,
      "eval_runtime": 4.9098,
      "eval_samples_per_second": 4.888,
      "eval_steps_per_second": 0.204,
      "step": 200
    },
    {
      "epoch": 0.64,
      "grad_norm": 31.32678074756829,
      "learning_rate": 1.7274575140626315e-07,
      "logits/chosen": -2.630878448486328,
      "logits/rejected": -2.360283136367798,
      "logps/chosen": -1274.6114501953125,
      "logps/rejected": -2422.227294921875,
      "loss": 0.107,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.9166343212127686,
      "rewards/margins": 7.893813133239746,
      "rewards/rejected": -10.810447692871094,
      "step": 210
    },
    {
      "epoch": 0.67,
      "grad_norm": 48.30372175049827,
      "learning_rate": 1.4791063411799937e-07,
      "logits/chosen": -2.758852481842041,
      "logits/rejected": -2.4495952129364014,
      "logps/chosen": -1242.4476318359375,
      "logps/rejected": -2313.494384765625,
      "loss": 0.1167,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.645324230194092,
      "rewards/margins": 7.094490051269531,
      "rewards/rejected": -9.739813804626465,
      "step": 220
    },
    {
      "epoch": 0.7,
      "grad_norm": 31.566932885659007,
      "learning_rate": 1.2423223013801945e-07,
      "logits/chosen": -2.7269630432128906,
      "logits/rejected": -2.492689609527588,
      "logps/chosen": -1234.9451904296875,
      "logps/rejected": -2318.334228515625,
      "loss": 0.1129,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.171018123626709,
      "rewards/margins": 7.9699273109436035,
      "rewards/rejected": -10.140945434570312,
      "step": 230
    },
    {
      "epoch": 0.73,
      "grad_norm": 34.83394111035656,
      "learning_rate": 1.0197882524482667e-07,
      "logits/chosen": -2.7251534461975098,
      "logits/rejected": -2.4403250217437744,
      "logps/chosen": -1231.4935302734375,
      "logps/rejected": -2440.34912109375,
      "loss": 0.1418,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.6753926277160645,
      "rewards/margins": 7.822265625,
      "rewards/rejected": -10.497659683227539,
      "step": 240
    },
    {
      "epoch": 0.76,
      "grad_norm": 24.25621896047951,
      "learning_rate": 8.140255940787058e-08,
      "logits/chosen": -2.695079803466797,
      "logits/rejected": -2.338381290435791,
      "logps/chosen": -1329.83642578125,
      "logps/rejected": -2476.803955078125,
      "loss": 0.1016,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.7425146102905273,
      "rewards/margins": 8.260651588439941,
      "rewards/rejected": -11.003166198730469,
      "step": 250
    },
    {
      "epoch": 0.79,
      "grad_norm": 27.532348797533775,
      "learning_rate": 6.273656994094231e-08,
      "logits/chosen": -2.669456958770752,
      "logits/rejected": -2.3488214015960693,
      "logps/chosen": -1346.739013671875,
      "logps/rejected": -2527.728759765625,
      "loss": 0.0974,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.164280891418457,
      "rewards/margins": 8.543807983398438,
      "rewards/rejected": -11.708089828491211,
      "step": 260
    },
    {
      "epoch": 0.82,
      "grad_norm": 32.33656052797523,
      "learning_rate": 4.6192349963253137e-08,
      "logits/chosen": -2.675199031829834,
      "logits/rejected": -2.319917678833008,
      "logps/chosen": -1294.7154541015625,
      "logps/rejected": -2497.270263671875,
      "loss": 0.0645,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.0818674564361572,
      "rewards/margins": 7.933001518249512,
      "rewards/rejected": -11.014870643615723,
      "step": 270
    },
    {
      "epoch": 0.85,
      "grad_norm": 29.09098674295539,
      "learning_rate": 3.195735209788528e-08,
      "logits/chosen": -2.6060080528259277,
      "logits/rejected": -2.3255152702331543,
      "logps/chosen": -1324.2171630859375,
      "logps/rejected": -2495.132080078125,
      "loss": 0.0796,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.3014044761657715,
      "rewards/margins": 7.963587760925293,
      "rewards/rejected": -11.264991760253906,
      "step": 280
    },
    {
      "epoch": 0.88,
      "grad_norm": 50.71746791104581,
      "learning_rate": 2.019286455866981e-08,
      "logits/chosen": -2.5728938579559326,
      "logits/rejected": -2.3195109367370605,
      "logps/chosen": -1388.82861328125,
      "logps/rejected": -2512.4306640625,
      "loss": 0.0931,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.2404701709747314,
      "rewards/margins": 8.0997953414917,
      "rewards/rejected": -11.340266227722168,
      "step": 290
    },
    {
      "epoch": 0.91,
      "grad_norm": 52.710422459009,
      "learning_rate": 1.1032183690276753e-08,
      "logits/chosen": -2.529834270477295,
      "logits/rejected": -2.2143425941467285,
      "logps/chosen": -1260.817626953125,
      "logps/rejected": -2584.9921875,
      "loss": 0.0848,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.3079543113708496,
      "rewards/margins": 9.132894515991211,
      "rewards/rejected": -12.440850257873535,
      "step": 300
    },
    {
      "epoch": 0.91,
      "eval_logits/chosen": -1.9181619882583618,
      "eval_logits/rejected": -1.5808876752853394,
      "eval_logps/chosen": -168.04290771484375,
      "eval_logps/rejected": -2051.701416015625,
      "eval_loss": 0.04682515189051628,
      "eval_rewards/accuracies": 0.875,
      "eval_rewards/chosen": -0.7745410203933716,
      "eval_rewards/margins": 13.737937927246094,
      "eval_rewards/rejected": -14.512479782104492,
      "eval_runtime": 4.9321,
      "eval_samples_per_second": 4.866,
      "eval_steps_per_second": 0.203,
      "step": 300
    },
    {
      "epoch": 0.95,
      "grad_norm": 72.2830720687719,
      "learning_rate": 4.579103667367384e-09,
      "logits/chosen": -2.5878891944885254,
      "logits/rejected": -2.2834665775299072,
      "logps/chosen": -1402.7425537109375,
      "logps/rejected": -2531.157958984375,
      "loss": 0.0942,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.4184463024139404,
      "rewards/margins": 8.603601455688477,
      "rewards/rejected": -12.022048950195312,
      "step": 310
    },
    {
      "epoch": 0.98,
      "grad_norm": 30.83428232415201,
      "learning_rate": 9.067404651211807e-10,
      "logits/chosen": -2.603289842605591,
      "logits/rejected": -2.298339366912842,
      "logps/chosen": -1401.251708984375,
      "logps/rejected": -2568.3681640625,
      "loss": 0.1105,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.375328779220581,
      "rewards/margins": 8.309103965759277,
      "rewards/rejected": -11.684433937072754,
      "step": 320
    },
    {
      "epoch": 1.0,
      "step": 328,
      "total_flos": 0.0,
      "train_loss": 0.23545521443209996,
      "train_runtime": 6883.5239,
      "train_samples_per_second": 3.046,
      "train_steps_per_second": 0.048
    }
  ],
  "logging_steps": 10,
  "max_steps": 328,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}