trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.9994765768123528,
  "eval_steps": 100,
  "global_step": 3820,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.3089005235602096e-08,
      "logits/chosen": 0.896942138671875,
      "logits/rejected": 0.9175108075141907,
      "logps/chosen": -192.32028198242188,
      "logps/rejected": -193.69876098632812,
      "loss": 2500.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 1.3089005235602095e-07,
      "logits/chosen": 0.903715968132019,
      "logits/rejected": 0.9309377670288086,
      "logps/chosen": -253.598876953125,
      "logps/rejected": -228.25482177734375,
      "loss": 2504.6897,
      "rewards/accuracies": 0.3819444477558136,
      "rewards/chosen": -0.0001807510998332873,
      "rewards/margins": -0.0004412428825162351,
      "rewards/rejected": 0.00026049179723486304,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.617801047120419e-07,
      "logits/chosen": 0.8256899118423462,
      "logits/rejected": 0.9293961524963379,
      "logps/chosen": -252.84963989257812,
      "logps/rejected": -214.4913330078125,
      "loss": 2511.0686,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.0004701187717728317,
      "rewards/margins": -0.0010407656664028764,
      "rewards/rejected": 0.0005706468946300447,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 3.926701570680629e-07,
      "logits/chosen": 0.8985889554023743,
      "logits/rejected": 0.8785662651062012,
      "logps/chosen": -236.40536499023438,
      "logps/rejected": -219.20285034179688,
      "loss": 2494.8072,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.0004474873130675405,
      "rewards/margins": 0.0005769692361354828,
      "rewards/rejected": -0.00012948190851602703,
      "step": 30
    },
    {
      "epoch": 0.02,
      "learning_rate": 5.235602094240838e-07,
      "logits/chosen": 0.819919228553772,
      "logits/rejected": 0.9144619703292847,
      "logps/chosen": -252.99588012695312,
      "logps/rejected": -225.9224853515625,
      "loss": 2504.7604,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.0003411176148802042,
      "rewards/margins": -0.0004235326196067035,
      "rewards/rejected": 8.241502655437216e-05,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 6.544502617801048e-07,
      "logits/chosen": 0.7974398136138916,
      "logits/rejected": 0.8803712725639343,
      "logps/chosen": -254.3247528076172,
      "logps/rejected": -243.318603515625,
      "loss": 2498.3947,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.00023676609271205962,
      "rewards/margins": 0.0002204025659011677,
      "rewards/rejected": -0.00045716846943832934,
      "step": 50
    },
    {
      "epoch": 0.03,
      "learning_rate": 7.853403141361258e-07,
      "logits/chosen": 0.8626053929328918,
      "logits/rejected": 0.8485649824142456,
      "logps/chosen": -262.6585693359375,
      "logps/rejected": -248.63272094726562,
      "loss": 2500.4902,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.0002292692952323705,
      "rewards/margins": 2.0124425645917654e-05,
      "rewards/rejected": -0.0002493937499821186,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 9.162303664921466e-07,
      "logits/chosen": 0.8897444605827332,
      "logits/rejected": 0.8922082185745239,
      "logps/chosen": -232.531005859375,
      "logps/rejected": -234.0869903564453,
      "loss": 2496.8041,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 3.7896970752626657e-06,
      "rewards/margins": 0.00037148987757973373,
      "rewards/rejected": -0.00036770018050447106,
      "step": 70
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.0471204188481676e-06,
      "logits/chosen": 0.8788009881973267,
      "logits/rejected": 0.8891068696975708,
      "logps/chosen": -242.5009765625,
      "logps/rejected": -229.5125732421875,
      "loss": 2508.6898,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.0007867829990573227,
      "rewards/margins": -0.0008174808463081717,
      "rewards/rejected": 3.069788363063708e-05,
      "step": 80
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.1780104712041885e-06,
      "logits/chosen": 0.8606799840927124,
      "logits/rejected": 0.9575719833374023,
      "logps/chosen": -232.0597686767578,
      "logps/rejected": -218.4732666015625,
      "loss": 2496.8559,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0003882866003550589,
      "rewards/margins": 0.000359431782271713,
      "rewards/rejected": 2.8854870834038593e-05,
      "step": 90
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.3089005235602096e-06,
      "logits/chosen": 0.8833224177360535,
      "logits/rejected": 0.8661258816719055,
      "logps/chosen": -245.799072265625,
      "logps/rejected": -249.2645721435547,
      "loss": 2496.843,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0005279771285131574,
      "rewards/margins": 0.00037464461638592184,
      "rewards/rejected": -0.0009026216575875878,
      "step": 100
    },
    {
      "epoch": 0.05,
      "eval_logits/chosen": 0.8318074345588684,
      "eval_logits/rejected": 0.8888298273086548,
      "eval_logps/chosen": -256.65057373046875,
      "eval_logps/rejected": -233.56494140625,
      "eval_loss": 2502.266845703125,
      "eval_rewards/accuracies": 0.5005000233650208,
      "eval_rewards/chosen": -0.0003313073539175093,
      "eval_rewards/margins": -0.00017098072567023337,
      "eval_rewards/rejected": -0.00016032661369536072,
      "eval_runtime": 416.835,
      "eval_samples_per_second": 4.798,
      "eval_steps_per_second": 1.2,
      "step": 100
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.4397905759162306e-06,
      "logits/chosen": 0.9012953042984009,
      "logits/rejected": 0.8766192197799683,
      "logps/chosen": -229.46292114257812,
      "logps/rejected": -210.2642364501953,
      "loss": 2501.3449,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00010574392217677087,
      "rewards/margins": -7.803810149198398e-05,
      "rewards/rejected": -2.7705809770850465e-05,
      "step": 110
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.5706806282722515e-06,
      "logits/chosen": 0.8108441233634949,
      "logits/rejected": 0.8906086087226868,
      "logps/chosen": -273.14385986328125,
      "logps/rejected": -259.1924133300781,
      "loss": 2494.2678,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.00021679741621483117,
      "rewards/margins": 0.0006280258530750871,
      "rewards/rejected": -0.00041122836410067976,
      "step": 120
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.7015706806282726e-06,
      "logits/chosen": 0.8712674975395203,
      "logits/rejected": 0.9049458503723145,
      "logps/chosen": -277.8616943359375,
      "logps/rejected": -222.53662109375,
      "loss": 2489.5006,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.0006732499459758401,
      "rewards/margins": 0.001107201213017106,
      "rewards/rejected": -0.0004339513252489269,
      "step": 130
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.8324607329842933e-06,
      "logits/chosen": 0.7955681085586548,
      "logits/rejected": 0.8811987638473511,
      "logps/chosen": -248.83865356445312,
      "logps/rejected": -246.317138671875,
      "loss": 2504.0979,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.00031032637343741953,
      "rewards/margins": -0.00034084441722370684,
      "rewards/rejected": 3.0518032872350886e-05,
      "step": 140
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.9633507853403143e-06,
      "logits/chosen": 0.7933157086372375,
      "logits/rejected": 0.8591764569282532,
      "logps/chosen": -257.7363586425781,
      "logps/rejected": -217.54580688476562,
      "loss": 2507.8082,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.0007291415822692215,
      "rewards/margins": -0.0007318807765841484,
      "rewards/rejected": 2.739173851296073e-06,
      "step": 150
    },
    {
      "epoch": 0.08,
      "learning_rate": 2.094240837696335e-06,
      "logits/chosen": 0.8041954040527344,
      "logits/rejected": 0.8887465596199036,
      "logps/chosen": -276.43304443359375,
      "logps/rejected": -250.4193572998047,
      "loss": 2504.2807,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00010556764755165204,
      "rewards/margins": -0.0003692187019623816,
      "rewards/rejected": 0.00026365104713477194,
      "step": 160
    },
    {
      "epoch": 0.09,
      "learning_rate": 2.2251308900523565e-06,
      "logits/chosen": 0.8059272766113281,
      "logits/rejected": 0.8950363993644714,
      "logps/chosen": -274.240234375,
      "logps/rejected": -247.8701171875,
      "loss": 2501.8248,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.001084670191630721,
      "rewards/margins": -0.0001141707762144506,
      "rewards/rejected": -0.0009704994154162705,
      "step": 170
    },
    {
      "epoch": 0.09,
      "learning_rate": 2.356020942408377e-06,
      "logits/chosen": 0.8783141374588013,
      "logits/rejected": 0.8253491520881653,
      "logps/chosen": -242.3585968017578,
      "logps/rejected": -221.0929718017578,
      "loss": 2496.3063,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.0004814372514374554,
      "rewards/margins": 0.0004220888367854059,
      "rewards/rejected": -0.0009035261464305222,
      "step": 180
    },
    {
      "epoch": 0.1,
      "learning_rate": 2.4869109947643982e-06,
      "logits/chosen": 0.8767743110656738,
      "logits/rejected": 0.8822822570800781,
      "logps/chosen": -246.2511444091797,
      "logps/rejected": -224.4364471435547,
      "loss": 2498.3156,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.00032182232826016843,
      "rewards/margins": 0.00022237170196603984,
      "rewards/rejected": 9.945056081051007e-05,
      "step": 190
    },
    {
      "epoch": 0.1,
      "learning_rate": 2.617801047120419e-06,
      "logits/chosen": 0.855573296546936,
      "logits/rejected": 0.9106731414794922,
      "logps/chosen": -258.25885009765625,
      "logps/rejected": -236.6140594482422,
      "loss": 2499.2807,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.00018535128037910908,
      "rewards/margins": 0.0001189738031825982,
      "rewards/rejected": -0.0003043250762857497,
      "step": 200
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": 0.8310006856918335,
      "eval_logits/rejected": 0.8882209062576294,
      "eval_logps/chosen": -256.6106262207031,
      "eval_logps/rejected": -233.5994873046875,
      "eval_loss": 2494.83544921875,
      "eval_rewards/accuracies": 0.5189999938011169,
      "eval_rewards/chosen": 6.786447193007916e-05,
      "eval_rewards/margins": 0.0005738017498515546,
      "eval_rewards/rejected": -0.0005059372633695602,
      "eval_runtime": 416.4863,
      "eval_samples_per_second": 4.802,
      "eval_steps_per_second": 1.201,
      "step": 200
    },
    {
      "epoch": 0.11,
      "learning_rate": 2.74869109947644e-06,
      "logits/chosen": 0.9517833590507507,
      "logits/rejected": 0.910740852355957,
      "logps/chosen": -244.80032348632812,
      "logps/rejected": -232.45321655273438,
      "loss": 2500.5746,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -9.258640056941658e-05,
      "rewards/margins": -6.444106020353502e-06,
      "rewards/rejected": -8.61423322930932e-05,
      "step": 210
    },
    {
      "epoch": 0.12,
      "learning_rate": 2.8795811518324613e-06,
      "logits/chosen": 0.8369059562683105,
      "logits/rejected": 0.8937622904777527,
      "logps/chosen": -267.40948486328125,
      "logps/rejected": -209.13290405273438,
      "loss": 2487.1783,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.0004562476242426783,
      "rewards/margins": 0.0013524172827601433,
      "rewards/rejected": -0.0008961696876212955,
      "step": 220
    },
    {
      "epoch": 0.12,
      "learning_rate": 3.010471204188482e-06,
      "logits/chosen": 0.8930699229240417,
      "logits/rejected": 0.9343907237052917,
      "logps/chosen": -258.6376037597656,
      "logps/rejected": -221.6857452392578,
      "loss": 2493.877,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": 0.0002062669227598235,
      "rewards/margins": 0.0006668218411505222,
      "rewards/rejected": -0.00046055493294261396,
      "step": 230
    },
    {
      "epoch": 0.13,
      "learning_rate": 3.141361256544503e-06,
      "logits/chosen": 0.9376400113105774,
      "logits/rejected": 0.8995400667190552,
      "logps/chosen": -228.9315185546875,
      "logps/rejected": -242.112548828125,
      "loss": 2480.1844,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0004402367048896849,
      "rewards/margins": 0.002044759690761566,
      "rewards/rejected": -0.001604523160494864,
      "step": 240
    },
    {
      "epoch": 0.13,
      "learning_rate": 3.2722513089005235e-06,
      "logits/chosen": 0.9079924821853638,
      "logits/rejected": 0.8767238855361938,
      "logps/chosen": -242.71121215820312,
      "logps/rejected": -233.9228973388672,
      "loss": 2493.2939,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -6.936644058441743e-05,
      "rewards/margins": 0.0007372990949079394,
      "rewards/rejected": -0.0008066653972491622,
      "step": 250
    },
    {
      "epoch": 0.14,
      "learning_rate": 3.403141361256545e-06,
      "logits/chosen": 0.8814166784286499,
      "logits/rejected": 0.9410937428474426,
      "logps/chosen": -236.88095092773438,
      "logps/rejected": -223.930419921875,
      "loss": 2496.3828,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.00017086375737562776,
      "rewards/margins": 0.0004164519195910543,
      "rewards/rejected": -0.0005873156478628516,
      "step": 260
    },
    {
      "epoch": 0.14,
      "learning_rate": 3.534031413612566e-06,
      "logits/chosen": 0.8372557759284973,
      "logits/rejected": 0.8741558194160461,
      "logps/chosen": -212.303466796875,
      "logps/rejected": -239.15158081054688,
      "loss": 2483.2088,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.00031574201420880854,
      "rewards/margins": 0.0017516377847641706,
      "rewards/rejected": -0.0020673798862844706,
      "step": 270
    },
    {
      "epoch": 0.15,
      "learning_rate": 3.6649214659685865e-06,
      "logits/chosen": 0.8835927844047546,
      "logits/rejected": 0.9312320947647095,
      "logps/chosen": -250.3417510986328,
      "logps/rejected": -263.51531982421875,
      "loss": 2492.1223,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0005325328675098717,
      "rewards/margins": 0.000863347842823714,
      "rewards/rejected": -0.0013958807103335857,
      "step": 280
    },
    {
      "epoch": 0.15,
      "learning_rate": 3.7958115183246074e-06,
      "logits/chosen": 0.8323150873184204,
      "logits/rejected": 0.8896921277046204,
      "logps/chosen": -250.1656951904297,
      "logps/rejected": -234.757568359375,
      "loss": 2483.249,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.00013801059685647488,
      "rewards/margins": 0.0017679758602753282,
      "rewards/rejected": -0.001905986457131803,
      "step": 290
    },
    {
      "epoch": 0.16,
      "learning_rate": 3.926701570680629e-06,
      "logits/chosen": 0.8511127233505249,
      "logits/rejected": 0.8209661245346069,
      "logps/chosen": -273.7172546386719,
      "logps/rejected": -250.82748413085938,
      "loss": 2477.7609,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0011967200553044677,
      "rewards/margins": 0.0023162723518908024,
      "rewards/rejected": -0.0035129922907799482,
      "step": 300
    },
    {
      "epoch": 0.16,
      "eval_logits/chosen": 0.8318725824356079,
      "eval_logits/rejected": 0.8892252445220947,
      "eval_logps/chosen": -256.7284851074219,
      "eval_logps/rejected": -233.8547821044922,
      "eval_loss": 2481.50146484375,
      "eval_rewards/accuracies": 0.559499979019165,
      "eval_rewards/chosen": -0.001110685057938099,
      "eval_rewards/margins": 0.0019479345064610243,
      "eval_rewards/rejected": -0.003058619564399123,
      "eval_runtime": 416.6935,
      "eval_samples_per_second": 4.8,
      "eval_steps_per_second": 1.2,
      "step": 300
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.05759162303665e-06,
      "logits/chosen": 0.7432538866996765,
      "logits/rejected": 0.817090630531311,
      "logps/chosen": -274.85931396484375,
      "logps/rejected": -236.4228057861328,
      "loss": 2469.5742,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.001123375492170453,
      "rewards/margins": 0.003155052661895752,
      "rewards/rejected": -0.004278427921235561,
      "step": 310
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.18848167539267e-06,
      "logits/chosen": 0.9147623777389526,
      "logits/rejected": 0.9334859848022461,
      "logps/chosen": -233.425537109375,
      "logps/rejected": -214.6092071533203,
      "loss": 2457.892,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.00017569802002981305,
      "rewards/margins": 0.004348465241491795,
      "rewards/rejected": -0.004172767512500286,
      "step": 320
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.319371727748692e-06,
      "logits/chosen": 0.8749006390571594,
      "logits/rejected": 0.9252738952636719,
      "logps/chosen": -247.8308563232422,
      "logps/rejected": -218.9490203857422,
      "loss": 2463.7824,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0017152890795841813,
      "rewards/margins": 0.0037990615237504244,
      "rewards/rejected": -0.0055143507197499275,
      "step": 330
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.450261780104713e-06,
      "logits/chosen": 0.8781224489212036,
      "logits/rejected": 0.9259663820266724,
      "logps/chosen": -253.4806365966797,
      "logps/rejected": -239.67434692382812,
      "loss": 2474.3055,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.0034939595498144627,
      "rewards/margins": 0.0027302266098558903,
      "rewards/rejected": -0.00622418662533164,
      "step": 340
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.5811518324607335e-06,
      "logits/chosen": 0.7855554223060608,
      "logits/rejected": 0.9314893484115601,
      "logps/chosen": -255.0915985107422,
      "logps/rejected": -206.8708038330078,
      "loss": 2432.458,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0009129707468673587,
      "rewards/margins": 0.007046517916023731,
      "rewards/rejected": -0.007959488779306412,
      "step": 350
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.712041884816754e-06,
      "logits/chosen": 0.8957219123840332,
      "logits/rejected": 0.8874330520629883,
      "logps/chosen": -257.02764892578125,
      "logps/rejected": -230.2834014892578,
      "loss": 2441.5672,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.001695218845270574,
      "rewards/margins": 0.006147631909698248,
      "rewards/rejected": -0.007842850871384144,
      "step": 360
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.842931937172775e-06,
      "logits/chosen": 0.9125442504882812,
      "logits/rejected": 0.8982815742492676,
      "logps/chosen": -230.16940307617188,
      "logps/rejected": -211.3495635986328,
      "loss": 2439.5756,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.004968739114701748,
      "rewards/margins": 0.006435071583837271,
      "rewards/rejected": -0.011403810232877731,
      "step": 370
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.9738219895287965e-06,
      "logits/chosen": 0.9134615063667297,
      "logits/rejected": 0.8667083978652954,
      "logps/chosen": -267.7635192871094,
      "logps/rejected": -219.404541015625,
      "loss": 2438.1262,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.004255574196577072,
      "rewards/margins": 0.006601777859032154,
      "rewards/rejected": -0.0108573529869318,
      "step": 380
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.999933200062888e-06,
      "logits/chosen": 0.8681972622871399,
      "logits/rejected": 0.8684479594230652,
      "logps/chosen": -253.1089324951172,
      "logps/rejected": -232.1811981201172,
      "loss": 2414.8473,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0038361139595508575,
      "rewards/margins": 0.009047028608620167,
      "rewards/rejected": -0.012883143499493599,
      "step": 390
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.999661831436499e-06,
      "logits/chosen": 0.9156022071838379,
      "logits/rejected": 0.9197471737861633,
      "logps/chosen": -260.40093994140625,
      "logps/rejected": -238.50961303710938,
      "loss": 2428.4195,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.005848343018442392,
      "rewards/margins": 0.007789201103150845,
      "rewards/rejected": -0.013637542724609375,
      "step": 400
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": 0.8404272794723511,
      "eval_logits/rejected": 0.8983384966850281,
      "eval_logps/chosen": -257.29510498046875,
      "eval_logps/rejected": -235.11265563964844,
      "eval_loss": 2419.1044921875,
      "eval_rewards/accuracies": 0.6495000123977661,
      "eval_rewards/chosen": -0.006776793394237757,
      "eval_rewards/margins": 0.008860657922923565,
      "eval_rewards/rejected": -0.01563744992017746,
      "eval_runtime": 416.4578,
      "eval_samples_per_second": 4.802,
      "eval_steps_per_second": 1.201,
      "step": 400
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.999181741766532e-06,
      "logits/chosen": 0.8992105722427368,
      "logits/rejected": 0.8969219923019409,
      "logps/chosen": -252.7702178955078,
      "logps/rejected": -249.8018798828125,
      "loss": 2438.1242,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.007479649968445301,
      "rewards/margins": 0.006951476447284222,
      "rewards/rejected": -0.014431129209697247,
      "step": 410
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.9984929711403395e-06,
      "logits/chosen": 0.913814902305603,
      "logits/rejected": 0.9069592356681824,
      "logps/chosen": -255.3079833984375,
      "logps/rejected": -251.0774383544922,
      "loss": 2417.1992,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.011085378006100655,
      "rewards/margins": 0.009236546233296394,
      "rewards/rejected": -0.020321926102042198,
      "step": 420
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.997595577070068e-06,
      "logits/chosen": 0.8943805694580078,
      "logits/rejected": 0.8994030952453613,
      "logps/chosen": -235.75009155273438,
      "logps/rejected": -232.7864990234375,
      "loss": 2429.1924,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.012942090630531311,
      "rewards/margins": 0.007976246066391468,
      "rewards/rejected": -0.020918335765600204,
      "step": 430
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.996489634487865e-06,
      "logits/chosen": 0.8081871867179871,
      "logits/rejected": 0.8945296406745911,
      "logps/chosen": -231.6891632080078,
      "logps/rejected": -253.6363067626953,
      "loss": 2484.0605,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.012701654806733131,
      "rewards/margins": 0.003084682859480381,
      "rewards/rejected": -0.015786338597536087,
      "step": 440
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.995175235739619e-06,
      "logits/chosen": 0.8565770983695984,
      "logits/rejected": 0.8623224496841431,
      "logps/chosen": -264.3777770996094,
      "logps/rejected": -267.23712158203125,
      "loss": 2392.5188,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.011126170866191387,
      "rewards/margins": 0.011980591341853142,
      "rewards/rejected": -0.023106763139367104,
      "step": 450
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.9936524905772466e-06,
      "logits/chosen": 0.78350430727005,
      "logits/rejected": 0.8708200454711914,
      "logps/chosen": -267.3443298339844,
      "logps/rejected": -236.05459594726562,
      "loss": 2404.3129,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.015727100893855095,
      "rewards/margins": 0.010795451700687408,
      "rewards/rejected": -0.026522550731897354,
      "step": 460
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.991921526149529e-06,
      "logits/chosen": 0.9162321090698242,
      "logits/rejected": 0.9280640482902527,
      "logps/chosen": -256.3532409667969,
      "logps/rejected": -247.89315795898438,
      "loss": 2386.3984,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.01587381586432457,
      "rewards/margins": 0.01281227171421051,
      "rewards/rejected": -0.02868608757853508,
      "step": 470
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.9899824869915e-06,
      "logits/chosen": 0.8048622012138367,
      "logits/rejected": 0.8283928036689758,
      "logps/chosen": -246.1285858154297,
      "logps/rejected": -252.7605743408203,
      "loss": 2396.8398,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.021037602797150612,
      "rewards/margins": 0.012100132182240486,
      "rewards/rejected": -0.0331377312541008,
      "step": 480
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.987835535012371e-06,
      "logits/chosen": 0.8453197479248047,
      "logits/rejected": 0.86089026927948,
      "logps/chosen": -240.1754913330078,
      "logps/rejected": -229.0949249267578,
      "loss": 2350.6213,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.019980577751994133,
      "rewards/margins": 0.01681477203965187,
      "rewards/rejected": -0.03679535537958145,
      "step": 490
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.985480849482012e-06,
      "logits/chosen": 0.8507258296012878,
      "logits/rejected": 0.8831195831298828,
      "logps/chosen": -264.3097839355469,
      "logps/rejected": -267.3841552734375,
      "loss": 2296.8842,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.019409244880080223,
      "rewards/margins": 0.023494381457567215,
      "rewards/rejected": -0.04290362820029259,
      "step": 500
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": 0.8214389681816101,
      "eval_logits/rejected": 0.8805551528930664,
      "eval_logps/chosen": -259.0124206542969,
      "eval_logps/rejected": -237.73793029785156,
      "eval_loss": 2349.435791015625,
      "eval_rewards/accuracies": 0.656499981880188,
      "eval_rewards/chosen": -0.023950034752488136,
      "eval_rewards/margins": 0.017940117046236992,
      "eval_rewards/rejected": -0.04189015179872513,
      "eval_runtime": 416.5178,
      "eval_samples_per_second": 4.802,
      "eval_steps_per_second": 1.2,
      "step": 500
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.98291862701599e-06,
      "logits/chosen": 0.8344039916992188,
      "logits/rejected": 0.8795874714851379,
      "logps/chosen": -254.190673828125,
      "logps/rejected": -214.09396362304688,
      "loss": 2307.1967,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.02996075712144375,
      "rewards/margins": 0.022381700575351715,
      "rewards/rejected": -0.052342455834150314,
      "step": 510
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.980149081559142e-06,
      "logits/chosen": 0.8595240712165833,
      "logits/rejected": 0.903663158416748,
      "logps/chosen": -237.7533416748047,
      "logps/rejected": -241.7561492919922,
      "loss": 2330.1758,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02604847028851509,
      "rewards/margins": 0.020816484466195107,
      "rewards/rejected": -0.04686495289206505,
      "step": 520
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.977172444367718e-06,
      "logits/chosen": 0.8232777714729309,
      "logits/rejected": 0.8955798149108887,
      "logps/chosen": -248.8101806640625,
      "logps/rejected": -226.21786499023438,
      "loss": 2284.3357,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0257116612046957,
      "rewards/margins": 0.026299094781279564,
      "rewards/rejected": -0.052010755985975266,
      "step": 530
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.9739889639900655e-06,
      "logits/chosen": 0.8850505948066711,
      "logits/rejected": 0.9008530378341675,
      "logps/chosen": -260.18963623046875,
      "logps/rejected": -228.6940155029297,
      "loss": 2314.4127,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.028372693806886673,
      "rewards/margins": 0.02354586310684681,
      "rewards/rejected": -0.051918547600507736,
      "step": 540
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.9705989062458805e-06,
      "logits/chosen": 0.8566417694091797,
      "logits/rejected": 0.8558026552200317,
      "logps/chosen": -242.9883270263672,
      "logps/rejected": -247.76370239257812,
      "loss": 2339.4164,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.03327028453350067,
      "rewards/margins": 0.02076330967247486,
      "rewards/rejected": -0.054033588618040085,
      "step": 550
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.967002554204009e-06,
      "logits/chosen": 0.7901058793067932,
      "logits/rejected": 0.8357075452804565,
      "logps/chosen": -263.3518371582031,
      "logps/rejected": -250.97958374023438,
      "loss": 2302.1379,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.04014524444937706,
      "rewards/margins": 0.025143718346953392,
      "rewards/rejected": -0.0652889683842659,
      "step": 560
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.963200208158811e-06,
      "logits/chosen": 0.8461757898330688,
      "logits/rejected": 0.9372328519821167,
      "logps/chosen": -223.61373901367188,
      "logps/rejected": -212.1987762451172,
      "loss": 2320.3783,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.04337463900446892,
      "rewards/margins": 0.021725038066506386,
      "rewards/rejected": -0.06509967893362045,
      "step": 570
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.959192185605089e-06,
      "logits/chosen": 0.7897135615348816,
      "logits/rejected": 0.9126697778701782,
      "logps/chosen": -294.5254821777344,
      "logps/rejected": -232.0474395751953,
      "loss": 2167.2896,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.03270890563726425,
      "rewards/margins": 0.041936445981264114,
      "rewards/rejected": -0.07464535534381866,
      "step": 580
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.95497882121157e-06,
      "logits/chosen": 0.8067277669906616,
      "logits/rejected": 0.8418477773666382,
      "logps/chosen": -240.95639038085938,
      "logps/rejected": -217.8979949951172,
      "loss": 2249.4051,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04393559694290161,
      "rewards/margins": 0.03141217678785324,
      "rewards/rejected": -0.07534776628017426,
      "step": 590
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.950560466792969e-06,
      "logits/chosen": 0.8411356806755066,
      "logits/rejected": 0.8524805307388306,
      "logps/chosen": -241.7827606201172,
      "logps/rejected": -234.87985229492188,
      "loss": 2254.5846,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04195866733789444,
      "rewards/margins": 0.03153757005929947,
      "rewards/rejected": -0.07349623739719391,
      "step": 600
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": 0.7868022322654724,
      "eval_logits/rejected": 0.8478493094444275,
      "eval_logps/chosen": -261.86590576171875,
      "eval_logps/rejected": -241.83828735351562,
      "eval_loss": 2273.499267578125,
      "eval_rewards/accuracies": 0.6570000052452087,
      "eval_rewards/chosen": -0.05248467996716499,
      "eval_rewards/margins": 0.03040897473692894,
      "eval_rewards/rejected": -0.08289366215467453,
      "eval_runtime": 416.6239,
      "eval_samples_per_second": 4.8,
      "eval_steps_per_second": 1.2,
      "step": 600
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.945937491280611e-06,
      "logits/chosen": 0.7756252288818359,
      "logits/rejected": 0.8814484477043152,
      "logps/chosen": -245.0117950439453,
      "logps/rejected": -217.2014923095703,
      "loss": 2348.3988,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.05819234997034073,
      "rewards/margins": 0.021564457565546036,
      "rewards/rejected": -0.07975681126117706,
      "step": 610
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.9411102806916185e-06,
      "logits/chosen": 0.7903825640678406,
      "logits/rejected": 0.8663345575332642,
      "logps/chosen": -270.52239990234375,
      "logps/rejected": -254.25369262695312,
      "loss": 2181.5219,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05034894496202469,
      "rewards/margins": 0.042683206498622894,
      "rewards/rejected": -0.09303215146064758,
      "step": 620
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.9360792380966875e-06,
      "logits/chosen": 0.8880133628845215,
      "logits/rejected": 0.8636928796768188,
      "logps/chosen": -241.08279418945312,
      "logps/rejected": -220.8309783935547,
      "loss": 2263.642,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.05800148844718933,
      "rewards/margins": 0.0320889875292778,
      "rewards/rejected": -0.09009047597646713,
      "step": 630
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.930844783586424e-06,
      "logits/chosen": 0.8702048063278198,
      "logits/rejected": 0.8985433578491211,
      "logps/chosen": -240.00460815429688,
      "logps/rejected": -238.61227416992188,
      "loss": 2244.0928,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.05921437591314316,
      "rewards/margins": 0.032827965915203094,
      "rewards/rejected": -0.09204234182834625,
      "step": 640
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.925407354236279e-06,
      "logits/chosen": 0.8151038885116577,
      "logits/rejected": 0.8790351152420044,
      "logps/chosen": -241.45755004882812,
      "logps/rejected": -226.37667846679688,
      "loss": 2236.8199,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06360708922147751,
      "rewards/margins": 0.03408312052488327,
      "rewards/rejected": -0.09769020974636078,
      "step": 650
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.919767404070033e-06,
      "logits/chosen": 0.8537635803222656,
      "logits/rejected": 0.8909260034561157,
      "logps/chosen": -221.7104034423828,
      "logps/rejected": -206.9849395751953,
      "loss": 2317.7387,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07372693717479706,
      "rewards/margins": 0.02632719837129116,
      "rewards/rejected": -0.10005412995815277,
      "step": 660
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.913925404021905e-06,
      "logits/chosen": 0.8039971590042114,
      "logits/rejected": 0.8006811141967773,
      "logps/chosen": -240.97213745117188,
      "logps/rejected": -209.97219848632812,
      "loss": 2208.3297,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.07874181121587753,
      "rewards/margins": 0.03945617750287056,
      "rewards/rejected": -0.11819799244403839,
      "step": 670
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.907881841897216e-06,
      "logits/chosen": 0.8469578623771667,
      "logits/rejected": 0.8435947299003601,
      "logps/chosen": -257.61810302734375,
      "logps/rejected": -245.639892578125,
      "loss": 2260.2275,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07468362152576447,
      "rewards/margins": 0.03678290545940399,
      "rewards/rejected": -0.11146652698516846,
      "step": 680
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.901637222331665e-06,
      "logits/chosen": 0.7657278776168823,
      "logits/rejected": 0.7471415996551514,
      "logps/chosen": -259.5301513671875,
      "logps/rejected": -236.86032104492188,
      "loss": 2287.06,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.08623397350311279,
      "rewards/margins": 0.03296409547328949,
      "rewards/rejected": -0.11919806897640228,
      "step": 690
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.89519206674919e-06,
      "logits/chosen": 0.7977254390716553,
      "logits/rejected": 0.8500420451164246,
      "logps/chosen": -244.900634765625,
      "logps/rejected": -251.4091339111328,
      "loss": 2330.7787,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08788236975669861,
      "rewards/margins": 0.02753649279475212,
      "rewards/rejected": -0.11541886627674103,
      "step": 700
    },
    {
      "epoch": 0.37,
      "eval_logits/chosen": 0.7516666650772095,
      "eval_logits/rejected": 0.812827467918396,
      "eval_logps/chosen": -264.809326171875,
      "eval_logps/rejected": -245.7631378173828,
      "eval_loss": 2224.3349609375,
      "eval_rewards/accuracies": 0.6629999876022339,
      "eval_rewards/chosen": -0.08191882818937302,
      "eval_rewards/margins": 0.0402236245572567,
      "eval_rewards/rejected": -0.12214244902133942,
      "eval_runtime": 416.6908,
      "eval_samples_per_second": 4.8,
      "eval_steps_per_second": 1.2,
      "step": 700
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.8885469133184235e-06,
      "logits/chosen": 0.8586422204971313,
      "logits/rejected": 0.8278988003730774,
      "logps/chosen": -244.2412109375,
      "logps/rejected": -248.913818359375,
      "loss": 2265.9631,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0877029225230217,
      "rewards/margins": 0.03201908990740776,
      "rewards/rejected": -0.11972200870513916,
      "step": 710
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.881702316907769e-06,
      "logits/chosen": 0.853478729724884,
      "logits/rejected": 0.9013971090316772,
      "logps/chosen": -240.85440063476562,
      "logps/rejected": -233.39761352539062,
      "loss": 2238.1352,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08841414749622345,
      "rewards/margins": 0.03821689262986183,
      "rewards/rejected": -0.12663105130195618,
      "step": 720
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.874658849039054e-06,
      "logits/chosen": 0.6829933524131775,
      "logits/rejected": 0.7707113027572632,
      "logps/chosen": -274.42095947265625,
      "logps/rejected": -232.1612091064453,
      "loss": 2119.9938,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07465031743049622,
      "rewards/margins": 0.049049459397792816,
      "rewards/rejected": -0.12369978427886963,
      "step": 730
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.86741709783982e-06,
      "logits/chosen": 0.7617892026901245,
      "logits/rejected": 0.8164576292037964,
      "logps/chosen": -267.9073486328125,
      "logps/rejected": -240.621826171875,
      "loss": 2243.2557,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.085872121155262,
      "rewards/margins": 0.042114924639463425,
      "rewards/rejected": -0.1279870569705963,
      "step": 740
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.859977667994209e-06,
      "logits/chosen": 0.74756920337677,
      "logits/rejected": 0.8244425654411316,
      "logps/chosen": -255.57754516601562,
      "logps/rejected": -242.3626251220703,
      "loss": 2231.4938,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.09324190765619278,
      "rewards/margins": 0.04055342823266983,
      "rewards/rejected": -0.1337953507900238,
      "step": 750
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.852341180692471e-06,
      "logits/chosen": 0.7698923945426941,
      "logits/rejected": 0.7992275953292847,
      "logps/chosen": -256.96868896484375,
      "logps/rejected": -271.1941833496094,
      "loss": 2239.7266,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.08798633515834808,
      "rewards/margins": 0.0419507697224617,
      "rewards/rejected": -0.1299370974302292,
      "step": 760
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.844508273579097e-06,
      "logits/chosen": 0.803545355796814,
      "logits/rejected": 0.7743754982948303,
      "logps/chosen": -249.5584716796875,
      "logps/rejected": -238.66683959960938,
      "loss": 2204.2545,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.08735480904579163,
      "rewards/margins": 0.04183940216898918,
      "rewards/rejected": -0.1291942000389099,
      "step": 770
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.836479600699579e-06,
      "logits/chosen": 0.7211157083511353,
      "logits/rejected": 0.7573873400688171,
      "logps/chosen": -251.00076293945312,
      "logps/rejected": -248.5784454345703,
      "loss": 2175.5199,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.09977405518293381,
      "rewards/margins": 0.047359712421894073,
      "rewards/rejected": -0.14713376760482788,
      "step": 780
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.82825583244579e-06,
      "logits/chosen": 0.7303954362869263,
      "logits/rejected": 0.7423623204231262,
      "logps/chosen": -271.29888916015625,
      "logps/rejected": -255.95645141601562,
      "loss": 2097.0797,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.09688698500394821,
      "rewards/margins": 0.060234714299440384,
      "rewards/rejected": -0.1571216881275177,
      "step": 790
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.819837655500014e-06,
      "logits/chosen": 0.7324298620223999,
      "logits/rejected": 0.8285747766494751,
      "logps/chosen": -259.47650146484375,
      "logps/rejected": -261.0966491699219,
      "loss": 2223.6863,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1077154278755188,
      "rewards/margins": 0.04667884111404419,
      "rewards/rejected": -0.154394268989563,
      "step": 800
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": 0.6992308497428894,
      "eval_logits/rejected": 0.7610952854156494,
      "eval_logps/chosen": -266.70574951171875,
      "eval_logps/rejected": -248.42222595214844,
      "eval_loss": 2196.09912109375,
      "eval_rewards/accuracies": 0.6675000190734863,
      "eval_rewards/chosen": -0.1008833572268486,
      "eval_rewards/margins": 0.04784964770078659,
      "eval_rewards/rejected": -0.1487330049276352,
      "eval_runtime": 416.5458,
      "eval_samples_per_second": 4.801,
      "eval_steps_per_second": 1.2,
      "step": 800
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.811225772777603e-06,
      "logits/chosen": 0.8175959587097168,
      "logits/rejected": 0.7778623700141907,
      "logps/chosen": -281.6056823730469,
      "logps/rejected": -235.4732666015625,
      "loss": 2154.2184,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10602346807718277,
      "rewards/margins": 0.0522245354950428,
      "rewards/rejected": -0.15824799239635468,
      "step": 810
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.802420903368286e-06,
      "logits/chosen": 0.6645683646202087,
      "logits/rejected": 0.7504470944404602,
      "logps/chosen": -262.6244812011719,
      "logps/rejected": -246.62728881835938,
      "loss": 2070.7916,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08415599167346954,
      "rewards/margins": 0.06298204511404037,
      "rewards/rejected": -0.14713802933692932,
      "step": 820
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.793423782476125e-06,
      "logits/chosen": 0.7014611959457397,
      "logits/rejected": 0.7595884203910828,
      "logps/chosen": -261.1951599121094,
      "logps/rejected": -237.8996124267578,
      "loss": 2303.9654,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.11634109169244766,
      "rewards/margins": 0.03935299813747406,
      "rewards/rejected": -0.15569409728050232,
      "step": 830
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.784235161358124e-06,
      "logits/chosen": 0.7220847010612488,
      "logits/rejected": 0.8208295702934265,
      "logps/chosen": -274.5125427246094,
      "logps/rejected": -267.01275634765625,
      "loss": 2374.0922,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.11991143226623535,
      "rewards/margins": 0.03418232128024101,
      "rewards/rejected": -0.15409375727176666,
      "step": 840
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.774855807261504e-06,
      "logits/chosen": 0.771617591381073,
      "logits/rejected": 0.7759231925010681,
      "logps/chosen": -266.54156494140625,
      "logps/rejected": -232.45114135742188,
      "loss": 2085.9527,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.09387587755918503,
      "rewards/margins": 0.059171438217163086,
      "rewards/rejected": -0.1530473232269287,
      "step": 850
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.765286503359632e-06,
      "logits/chosen": 0.7542043924331665,
      "logits/rejected": 0.7199236154556274,
      "logps/chosen": -247.95004272460938,
      "logps/rejected": -234.8477325439453,
      "loss": 2140.2543,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.09680913388729095,
      "rewards/margins": 0.05759376287460327,
      "rewards/rejected": -0.15440289676189423,
      "step": 860
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.755528048686629e-06,
      "logits/chosen": 0.7054905891418457,
      "logits/rejected": 0.7627168297767639,
      "logps/chosen": -269.0238342285156,
      "logps/rejected": -231.31594848632812,
      "loss": 2018.9971,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.09635048359632492,
      "rewards/margins": 0.07080944627523422,
      "rewards/rejected": -0.16715992987155914,
      "step": 870
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.745581258070654e-06,
      "logits/chosen": 0.7587330937385559,
      "logits/rejected": 0.7288376092910767,
      "logps/chosen": -279.3045959472656,
      "logps/rejected": -248.2842559814453,
      "loss": 2182.2826,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.10375384986400604,
      "rewards/margins": 0.04985477030277252,
      "rewards/rejected": -0.15360862016677856,
      "step": 880
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.73544696206586e-06,
      "logits/chosen": 0.7099634408950806,
      "logits/rejected": 0.7651978731155396,
      "logps/chosen": -241.7110137939453,
      "logps/rejected": -226.1469268798828,
      "loss": 2313.518,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.11171738058328629,
      "rewards/margins": 0.033566057682037354,
      "rewards/rejected": -0.14528343081474304,
      "step": 890
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.725126006883047e-06,
      "logits/chosen": 0.6832414865493774,
      "logits/rejected": 0.6996358633041382,
      "logps/chosen": -287.02655029296875,
      "logps/rejected": -271.8457946777344,
      "loss": 2066.7418,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1082179993391037,
      "rewards/margins": 0.06335000693798065,
      "rewards/rejected": -0.17156800627708435,
      "step": 900
    },
    {
      "epoch": 0.47,
      "eval_logits/chosen": 0.6917389035224915,
      "eval_logits/rejected": 0.7518260478973389,
      "eval_logps/chosen": -267.73968505859375,
      "eval_logps/rejected": -250.13189697265625,
      "eval_loss": 2166.0732421875,
      "eval_rewards/accuracies": 0.6700000166893005,
      "eval_rewards/chosen": -0.11122233420610428,
      "eval_rewards/margins": 0.0546073243021965,
      "eval_rewards/rejected": -0.16582968831062317,
      "eval_runtime": 416.784,
      "eval_samples_per_second": 4.799,
      "eval_steps_per_second": 1.2,
      "step": 900
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.7146192543190005e-06,
      "logits/chosen": 0.7068012952804565,
      "logits/rejected": 0.7604703307151794,
      "logps/chosen": -301.75897216796875,
      "logps/rejected": -260.5961608886719,
      "loss": 2114.2752,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10466556251049042,
      "rewards/margins": 0.06352122128009796,
      "rewards/rejected": -0.16818679869174957,
      "step": 910
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.70392758168454e-06,
      "logits/chosen": 0.7013599872589111,
      "logits/rejected": 0.7524459362030029,
      "logps/chosen": -280.307861328125,
      "logps/rejected": -256.722900390625,
      "loss": 2081.7857,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11534447968006134,
      "rewards/margins": 0.06791369616985321,
      "rewards/rejected": -0.18325819075107574,
      "step": 920
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.693051881731251e-06,
      "logits/chosen": 0.6879482269287109,
      "logits/rejected": 0.7315651178359985,
      "logps/chosen": -267.4771728515625,
      "logps/rejected": -269.62249755859375,
      "loss": 2219.8021,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.11604435741901398,
      "rewards/margins": 0.04904730245471001,
      "rewards/rejected": -0.16509169340133667,
      "step": 930
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.68199306257695e-06,
      "logits/chosen": 0.7221347093582153,
      "logits/rejected": 0.8089338541030884,
      "logps/chosen": -277.0709533691406,
      "logps/rejected": -288.2813415527344,
      "loss": 2066.718,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.09484682977199554,
      "rewards/margins": 0.066488116979599,
      "rewards/rejected": -0.16133496165275574,
      "step": 940
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.670752047629855e-06,
      "logits/chosen": 0.7649358510971069,
      "logits/rejected": 0.8068546056747437,
      "logps/chosen": -289.7987976074219,
      "logps/rejected": -257.411865234375,
      "loss": 1989.5814,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09663524478673935,
      "rewards/margins": 0.07417033612728119,
      "rewards/rejected": -0.17080560326576233,
      "step": 950
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.659329775511478e-06,
      "logits/chosen": 0.6801126599311829,
      "logits/rejected": 0.7201008796691895,
      "logps/chosen": -275.82244873046875,
      "logps/rejected": -263.333251953125,
      "loss": 2137.0553,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.10409847646951675,
      "rewards/margins": 0.06283750385046005,
      "rewards/rejected": -0.1669359654188156,
      "step": 960
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.647727199978255e-06,
      "logits/chosen": 0.675479531288147,
      "logits/rejected": 0.755820095539093,
      "logps/chosen": -281.660888671875,
      "logps/rejected": -264.6354675292969,
      "loss": 2290.1266,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.11869573593139648,
      "rewards/margins": 0.0478622205555439,
      "rewards/rejected": -0.1665579378604889,
      "step": 970
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.635945289841902e-06,
      "logits/chosen": 0.6294044852256775,
      "logits/rejected": 0.701261043548584,
      "logps/chosen": -249.8705596923828,
      "logps/rejected": -245.52346801757812,
      "loss": 2214.392,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.12245059013366699,
      "rewards/margins": 0.05106619745492935,
      "rewards/rejected": -0.17351679503917694,
      "step": 980
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.623985028888527e-06,
      "logits/chosen": 0.7620214223861694,
      "logits/rejected": 0.799843430519104,
      "logps/chosen": -236.2934112548828,
      "logps/rejected": -222.5919189453125,
      "loss": 2149.6912,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.12090835720300674,
      "rewards/margins": 0.05957023426890373,
      "rewards/rejected": -0.18047860264778137,
      "step": 990
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.611847415796476e-06,
      "logits/chosen": 0.7124743461608887,
      "logits/rejected": 0.6904253363609314,
      "logps/chosen": -265.0976257324219,
      "logps/rejected": -254.4890594482422,
      "loss": 2119.2691,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11531393229961395,
      "rewards/margins": 0.0636182576417923,
      "rewards/rejected": -0.17893218994140625,
      "step": 1000
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": 0.6619382500648499,
      "eval_logits/rejected": 0.721328854560852,
      "eval_logps/chosen": -268.7693176269531,
      "eval_logps/rejected": -251.76100158691406,
      "eval_loss": 2138.93115234375,
      "eval_rewards/accuracies": 0.671500027179718,
      "eval_rewards/chosen": -0.12151883542537689,
      "eval_rewards/margins": 0.060602057725191116,
      "eval_rewards/rejected": -0.1821209043264389,
      "eval_runtime": 416.4897,
      "eval_samples_per_second": 4.802,
      "eval_steps_per_second": 1.201,
      "step": 1000
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.599533464052951e-06,
      "logits/chosen": 0.7095866203308105,
      "logits/rejected": 0.7142434120178223,
      "logps/chosen": -285.8958740234375,
      "logps/rejected": -269.59906005859375,
      "loss": 2002.4428,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.09995652735233307,
      "rewards/margins": 0.08161594718694687,
      "rewards/rejected": -0.18157246708869934,
      "step": 1010
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.587044201869378e-06,
      "logits/chosen": 0.6786423921585083,
      "logits/rejected": 0.7168447375297546,
      "logps/chosen": -285.7081298828125,
      "logps/rejected": -245.5774688720703,
      "loss": 2104.4162,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10604969412088394,
      "rewards/margins": 0.06783930957317352,
      "rewards/rejected": -0.17388899624347687,
      "step": 1020
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.574380672095555e-06,
      "logits/chosen": 0.6884575486183167,
      "logits/rejected": 0.7298802733421326,
      "logps/chosen": -223.19393920898438,
      "logps/rejected": -240.6363983154297,
      "loss": 2218.8305,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.1402120441198349,
      "rewards/margins": 0.049252741038799286,
      "rewards/rejected": -0.1894647777080536,
      "step": 1030
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.561543932132574e-06,
      "logits/chosen": 0.6861045360565186,
      "logits/rejected": 0.7231858968734741,
      "logps/chosen": -282.3648376464844,
      "logps/rejected": -247.057373046875,
      "loss": 2149.1324,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1236349567770958,
      "rewards/margins": 0.05487797409296036,
      "rewards/rejected": -0.17851293087005615,
      "step": 1040
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.548535053844527e-06,
      "logits/chosen": 0.6396581530570984,
      "logits/rejected": 0.7092006206512451,
      "logps/chosen": -280.1047058105469,
      "logps/rejected": -267.3292541503906,
      "loss": 2057.9182,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.12618432939052582,
      "rewards/margins": 0.07123459875583649,
      "rewards/rejected": -0.1974189579486847,
      "step": 1050
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.535355123469009e-06,
      "logits/chosen": 0.7520751357078552,
      "logits/rejected": 0.7533235549926758,
      "logps/chosen": -246.0561065673828,
      "logps/rejected": -210.5857391357422,
      "loss": 2157.4354,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.11811725795269012,
      "rewards/margins": 0.05882970616221428,
      "rewards/rejected": -0.1769469678401947,
      "step": 1060
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.522005241526411e-06,
      "logits/chosen": 0.670494019985199,
      "logits/rejected": 0.7469106912612915,
      "logps/chosen": -281.83013916015625,
      "logps/rejected": -235.8207244873047,
      "loss": 2224.4785,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12622186541557312,
      "rewards/margins": 0.04977993294596672,
      "rewards/rejected": -0.17600181698799133,
      "step": 1070
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.508486522728037e-06,
      "logits/chosen": 0.6910241842269897,
      "logits/rejected": 0.7257175445556641,
      "logps/chosen": -276.518798828125,
      "logps/rejected": -260.69781494140625,
      "loss": 2003.6445,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1153578907251358,
      "rewards/margins": 0.07534319162368774,
      "rewards/rejected": -0.19070109724998474,
      "step": 1080
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.494800095883014e-06,
      "logits/chosen": 0.6132059097290039,
      "logits/rejected": 0.6958727836608887,
      "logps/chosen": -290.1937255859375,
      "logps/rejected": -244.7795867919922,
      "loss": 1914.4395,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.1075458973646164,
      "rewards/margins": 0.09307406842708588,
      "rewards/rejected": -0.20061998069286346,
      "step": 1090
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.480947103804044e-06,
      "logits/chosen": 0.5848616361618042,
      "logits/rejected": 0.6844476461410522,
      "logps/chosen": -286.0004577636719,
      "logps/rejected": -235.7235565185547,
      "loss": 2191.7109,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.12465153634548187,
      "rewards/margins": 0.05475841090083122,
      "rewards/rejected": -0.1794099658727646,
      "step": 1100
    },
    {
      "epoch": 0.58,
      "eval_logits/chosen": 0.6583799123764038,
      "eval_logits/rejected": 0.7175658345222473,
      "eval_logps/chosen": -269.19097900390625,
      "eval_logps/rejected": -252.60589599609375,
      "eval_loss": 2121.8115234375,
      "eval_rewards/accuracies": 0.6694999933242798,
      "eval_rewards/chosen": -0.12573528289794922,
      "eval_rewards/margins": 0.06483451277017593,
      "eval_rewards/rejected": -0.19056977331638336,
      "eval_runtime": 416.4568,
      "eval_samples_per_second": 4.802,
      "eval_steps_per_second": 1.201,
      "step": 1100
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.466928703211981e-06,
      "logits/chosen": 0.6980951428413391,
      "logits/rejected": 0.6862035989761353,
      "logps/chosen": -281.24700927734375,
      "logps/rejected": -248.4276123046875,
      "loss": 2063.3184,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.11639384180307388,
      "rewards/margins": 0.07433497160673141,
      "rewards/rejected": -0.1907288283109665,
      "step": 1110
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.452746064639239e-06,
      "logits/chosen": 0.678636372089386,
      "logits/rejected": 0.6571283936500549,
      "logps/chosen": -258.97674560546875,
      "logps/rejected": -254.3134307861328,
      "loss": 2036.2967,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.10998505353927612,
      "rewards/margins": 0.06968870759010315,
      "rewards/rejected": -0.17967377603054047,
      "step": 1120
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.438400372332058e-06,
      "logits/chosen": 0.7093490958213806,
      "logits/rejected": 0.7673132419586182,
      "logps/chosen": -265.0453796386719,
      "logps/rejected": -250.15982055664062,
      "loss": 1899.6914,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.10779963433742523,
      "rewards/margins": 0.08556055277585983,
      "rewards/rejected": -0.19336020946502686,
      "step": 1130
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.423892824151617e-06,
      "logits/chosen": 0.6779216527938843,
      "logits/rejected": 0.7495394945144653,
      "logps/chosen": -276.6440734863281,
      "logps/rejected": -247.07150268554688,
      "loss": 2002.1414,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.11242429912090302,
      "rewards/margins": 0.08376909792423248,
      "rewards/rejected": -0.19619342684745789,
      "step": 1140
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.409224631474014e-06,
      "logits/chosen": 0.6950255632400513,
      "logits/rejected": 0.7308493256568909,
      "logps/chosen": -258.3533020019531,
      "logps/rejected": -235.54483032226562,
      "loss": 1995.5852,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.12844698131084442,
      "rewards/margins": 0.07693418860435486,
      "rewards/rejected": -0.20538118481636047,
      "step": 1150
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.3943970190891164e-06,
      "logits/chosen": 0.6389755010604858,
      "logits/rejected": 0.6700756549835205,
      "logps/chosen": -264.4709167480469,
      "logps/rejected": -256.2208557128906,
      "loss": 2016.508,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.13102933764457703,
      "rewards/margins": 0.07655525207519531,
      "rewards/rejected": -0.20758457481861115,
      "step": 1160
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.379411225098292e-06,
      "logits/chosen": 0.6980705261230469,
      "logits/rejected": 0.7923838496208191,
      "logps/chosen": -283.1957702636719,
      "logps/rejected": -267.30938720703125,
      "loss": 2012.2674,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.138292133808136,
      "rewards/margins": 0.08078579604625702,
      "rewards/rejected": -0.2190779149532318,
      "step": 1170
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.364268500811025e-06,
      "logits/chosen": 0.6915451288223267,
      "logits/rejected": 0.6770834922790527,
      "logps/chosen": -258.2509460449219,
      "logps/rejected": -266.7464294433594,
      "loss": 2101.8338,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13359954953193665,
      "rewards/margins": 0.07587826251983643,
      "rewards/rejected": -0.20947781205177307,
      "step": 1180
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.348970110640437e-06,
      "logits/chosen": 0.6509718298912048,
      "logits/rejected": 0.7129366993904114,
      "logps/chosen": -258.56280517578125,
      "logps/rejected": -237.4566192626953,
      "loss": 2027.6717,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.14995795488357544,
      "rewards/margins": 0.07759587466716766,
      "rewards/rejected": -0.2275538146495819,
      "step": 1190
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.333517331997704e-06,
      "logits/chosen": 0.5978332161903381,
      "logits/rejected": 0.6565033793449402,
      "logps/chosen": -272.02166748046875,
      "logps/rejected": -271.3028259277344,
      "loss": 2308.1883,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1387997567653656,
      "rewards/margins": 0.04619471728801727,
      "rewards/rejected": -0.18499447405338287,
      "step": 1200
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": 0.6329967379570007,
      "eval_logits/rejected": 0.6920445561408997,
      "eval_logps/chosen": -270.7044372558594,
      "eval_logps/rejected": -254.78115844726562,
      "eval_loss": 2110.306884765625,
      "eval_rewards/accuracies": 0.6664999723434448,
      "eval_rewards/chosen": -0.14087003469467163,
      "eval_rewards/margins": 0.07145243883132935,
      "eval_rewards/rejected": -0.21232248842716217,
      "eval_runtime": 416.6934,
      "eval_samples_per_second": 4.8,
      "eval_steps_per_second": 1.2,
      "step": 1200
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.317911455185396e-06,
      "logits/chosen": 0.6959893703460693,
      "logits/rejected": 0.7259203791618347,
      "logps/chosen": -266.06829833984375,
      "logps/rejected": -238.3871307373047,
      "loss": 2262.5527,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.15047064423561096,
      "rewards/margins": 0.04972488805651665,
      "rewards/rejected": -0.2001955509185791,
      "step": 1210
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.302153783289737e-06,
      "logits/chosen": 0.6275352239608765,
      "logits/rejected": 0.7193800806999207,
      "logps/chosen": -266.05908203125,
      "logps/rejected": -258.8971252441406,
      "loss": 2233.5896,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.1438441276550293,
      "rewards/margins": 0.05622429400682449,
      "rewards/rejected": -0.2000684291124344,
      "step": 1220
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.286245632071791e-06,
      "logits/chosen": 0.6443454623222351,
      "logits/rejected": 0.6870865225791931,
      "logps/chosen": -257.45611572265625,
      "logps/rejected": -253.72756958007812,
      "loss": 2126.4154,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.15624599158763885,
      "rewards/margins": 0.06528286635875702,
      "rewards/rejected": -0.22152885794639587,
      "step": 1230
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.270188329857613e-06,
      "logits/chosen": 0.7713927626609802,
      "logits/rejected": 0.7753847241401672,
      "logps/chosen": -263.57708740234375,
      "logps/rejected": -270.3426513671875,
      "loss": 2108.1736,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.12420948594808578,
      "rewards/margins": 0.06607901304960251,
      "rewards/rejected": -0.1902884989976883,
      "step": 1240
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.253983217427313e-06,
      "logits/chosen": 0.6878337860107422,
      "logits/rejected": 0.7090884447097778,
      "logps/chosen": -271.42657470703125,
      "logps/rejected": -288.615478515625,
      "loss": 2128.0695,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.13797307014465332,
      "rewards/margins": 0.06890544295310974,
      "rewards/rejected": -0.20687851309776306,
      "step": 1250
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.237631647903115e-06,
      "logits/chosen": 0.6635148525238037,
      "logits/rejected": 0.6455484628677368,
      "logps/chosen": -266.98248291015625,
      "logps/rejected": -250.23403930664062,
      "loss": 2209.6877,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.13750961422920227,
      "rewards/margins": 0.05613657087087631,
      "rewards/rejected": -0.1936461478471756,
      "step": 1260
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.221134986636371e-06,
      "logits/chosen": 0.6171488761901855,
      "logits/rejected": 0.6567360758781433,
      "logps/chosen": -273.8824157714844,
      "logps/rejected": -249.103515625,
      "loss": 1885.2906,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.11038468778133392,
      "rewards/margins": 0.10038020461797714,
      "rewards/rejected": -0.21076488494873047,
      "step": 1270
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.204494611093548e-06,
      "logits/chosen": 0.7011705636978149,
      "logits/rejected": 0.6791177988052368,
      "logps/chosen": -251.37478637695312,
      "logps/rejected": -261.1529846191406,
      "loss": 2258.1799,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.14569328725337982,
      "rewards/margins": 0.0609690323472023,
      "rewards/rejected": -0.20666229724884033,
      "step": 1280
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.1877119107412165e-06,
      "logits/chosen": 0.6343793869018555,
      "logits/rejected": 0.6927725672721863,
      "logps/chosen": -237.8370819091797,
      "logps/rejected": -256.54254150390625,
      "loss": 2061.6873,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1375429332256317,
      "rewards/margins": 0.07506345212459564,
      "rewards/rejected": -0.21260638535022736,
      "step": 1290
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.170788286930024e-06,
      "logits/chosen": 0.6356396675109863,
      "logits/rejected": 0.7518913149833679,
      "logps/chosen": -275.9583435058594,
      "logps/rejected": -253.1591033935547,
      "loss": 1996.7178,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1259593516588211,
      "rewards/margins": 0.09192151576280594,
      "rewards/rejected": -0.21788087487220764,
      "step": 1300
    },
    {
      "epoch": 0.68,
      "eval_logits/chosen": 0.6140788793563843,
      "eval_logits/rejected": 0.6721699833869934,
      "eval_logps/chosen": -269.7620544433594,
      "eval_logps/rejected": -253.97259521484375,
      "eval_loss": 2095.31298828125,
      "eval_rewards/accuracies": 0.6754999756813049,
      "eval_rewards/chosen": -0.1314462274312973,
      "eval_rewards/margins": 0.07279053330421448,
      "eval_rewards/rejected": -0.20423679053783417,
      "eval_runtime": 416.679,
      "eval_samples_per_second": 4.8,
      "eval_steps_per_second": 1.2,
      "step": 1300
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.15372515277769e-06,
      "logits/chosen": 0.6244436502456665,
      "logits/rejected": 0.6609630584716797,
      "logps/chosen": -280.32794189453125,
      "logps/rejected": -248.7582244873047,
      "loss": 2011.6244,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.11511021852493286,
      "rewards/margins": 0.08765153586864471,
      "rewards/rejected": -0.20276173949241638,
      "step": 1310
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.136523933051005e-06,
      "logits/chosen": 0.6894992589950562,
      "logits/rejected": 0.6712801456451416,
      "logps/chosen": -263.05767822265625,
      "logps/rejected": -234.242431640625,
      "loss": 1956.5369,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.11226965487003326,
      "rewards/margins": 0.09646574407815933,
      "rewards/rejected": -0.208735391497612,
      "step": 1320
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.119186064046868e-06,
      "logits/chosen": 0.6183528900146484,
      "logits/rejected": 0.644507110118866,
      "logps/chosen": -274.1941223144531,
      "logps/rejected": -245.35562133789062,
      "loss": 2166.2785,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.12718230485916138,
      "rewards/margins": 0.06536950916051865,
      "rewards/rejected": -0.19255182147026062,
      "step": 1330
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.101712993472348e-06,
      "logits/chosen": 0.6670488715171814,
      "logits/rejected": 0.6754225492477417,
      "logps/chosen": -278.2078552246094,
      "logps/rejected": -241.5122528076172,
      "loss": 1883.2881,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.119564950466156,
      "rewards/margins": 0.09150619804859161,
      "rewards/rejected": -0.2110711634159088,
      "step": 1340
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.084106180323813e-06,
      "logits/chosen": 0.6214176416397095,
      "logits/rejected": 0.66867595911026,
      "logps/chosen": -261.7587890625,
      "logps/rejected": -251.6807098388672,
      "loss": 2017.3074,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1294572651386261,
      "rewards/margins": 0.08191975206136703,
      "rewards/rejected": -0.21137702465057373,
      "step": 1350
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.066367094765091e-06,
      "logits/chosen": 0.6692546606063843,
      "logits/rejected": 0.6942587494850159,
      "logps/chosen": -259.8294982910156,
      "logps/rejected": -268.8597412109375,
      "loss": 2097.6473,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.12980519235134125,
      "rewards/margins": 0.07286903262138367,
      "rewards/rejected": -0.20267422497272491,
      "step": 1360
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.048497218004724e-06,
      "logits/chosen": 0.5632964968681335,
      "logits/rejected": 0.6666015386581421,
      "logps/chosen": -265.3409729003906,
      "logps/rejected": -252.78958129882812,
      "loss": 2076.2803,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13252495229244232,
      "rewards/margins": 0.08360429853200912,
      "rewards/rejected": -0.21612922847270966,
      "step": 1370
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.030498042172277e-06,
      "logits/chosen": 0.6174412369728088,
      "logits/rejected": 0.6783226728439331,
      "logps/chosen": -249.213134765625,
      "logps/rejected": -241.413330078125,
      "loss": 2285.0457,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.14881116151809692,
      "rewards/margins": 0.05003712326288223,
      "rewards/rejected": -0.19884827733039856,
      "step": 1380
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.012371070193753e-06,
      "logits/chosen": 0.6269063353538513,
      "logits/rejected": 0.6346549391746521,
      "logps/chosen": -241.5870361328125,
      "logps/rejected": -245.96456909179688,
      "loss": 2133.8158,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.1412040889263153,
      "rewards/margins": 0.06111832335591316,
      "rewards/rejected": -0.20232239365577698,
      "step": 1390
    },
    {
      "epoch": 0.73,
      "learning_rate": 3.994117815666095e-06,
      "logits/chosen": 0.6533576250076294,
      "logits/rejected": 0.6740087866783142,
      "logps/chosen": -283.4432678222656,
      "logps/rejected": -267.79071044921875,
      "loss": 2038.3844,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1330070197582245,
      "rewards/margins": 0.07623559981584549,
      "rewards/rejected": -0.20924265682697296,
      "step": 1400
    },
    {
      "epoch": 0.73,
      "eval_logits/chosen": 0.5933060050010681,
      "eval_logits/rejected": 0.6512511968612671,
      "eval_logps/chosen": -270.4488220214844,
      "eval_logps/rejected": -254.944091796875,
      "eval_loss": 2085.085205078125,
      "eval_rewards/accuracies": 0.6800000071525574,
      "eval_rewards/chosen": -0.13831348717212677,
      "eval_rewards/margins": 0.07563827186822891,
      "eval_rewards/rejected": -0.21395176649093628,
      "eval_runtime": 416.609,
      "eval_samples_per_second": 4.801,
      "eval_steps_per_second": 1.2,
      "step": 1400
    },
    {
      "epoch": 0.74,
      "learning_rate": 3.975739802730805e-06,
      "logits/chosen": 0.5807250738143921,
      "logits/rejected": 0.6861320734024048,
      "logps/chosen": -298.25604248046875,
      "logps/rejected": -273.8665466308594,
      "loss": 2016.2207,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.12952394783496857,
      "rewards/margins": 0.08187790215015411,
      "rewards/rejected": -0.21140184998512268,
      "step": 1410
    },
    {
      "epoch": 0.74,
      "learning_rate": 3.957238565946672e-06,
      "logits/chosen": 0.6601132750511169,
      "logits/rejected": 0.6705759763717651,
      "logps/chosen": -266.2096252441406,
      "logps/rejected": -249.5722198486328,
      "loss": 2228.1479,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1489478349685669,
      "rewards/margins": 0.05832044407725334,
      "rewards/rejected": -0.20726828277111053,
      "step": 1420
    },
    {
      "epoch": 0.75,
      "learning_rate": 3.938615650161645e-06,
      "logits/chosen": 0.6056556701660156,
      "logits/rejected": 0.5954689979553223,
      "logps/chosen": -244.7415771484375,
      "logps/rejected": -234.8352813720703,
      "loss": 2023.5078,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.11950768530368805,
      "rewards/margins": 0.0772583931684494,
      "rewards/rejected": -0.19676607847213745,
      "step": 1430
    },
    {
      "epoch": 0.75,
      "learning_rate": 3.919872610383831e-06,
      "logits/chosen": 0.5716265439987183,
      "logits/rejected": 0.6326289176940918,
      "logps/chosen": -257.7333984375,
      "logps/rejected": -234.2956085205078,
      "loss": 2199.8311,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.14616943895816803,
      "rewards/margins": 0.055809132754802704,
      "rewards/rejected": -0.20197856426239014,
      "step": 1440
    },
    {
      "epoch": 0.76,
      "learning_rate": 3.9010110116516595e-06,
      "logits/chosen": 0.655591607093811,
      "logits/rejected": 0.7094139456748962,
      "logps/chosen": -266.137939453125,
      "logps/rejected": -247.2853240966797,
      "loss": 1965.8004,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.13736537098884583,
      "rewards/margins": 0.0867539569735527,
      "rewards/rejected": -0.22411933541297913,
      "step": 1450
    },
    {
      "epoch": 0.76,
      "learning_rate": 3.882032428903195e-06,
      "logits/chosen": 0.6148696541786194,
      "logits/rejected": 0.6896382570266724,
      "logps/chosen": -256.70904541015625,
      "logps/rejected": -245.3892822265625,
      "loss": 2095.5068,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.12945787608623505,
      "rewards/margins": 0.07017168402671814,
      "rewards/rejected": -0.19962957501411438,
      "step": 1460
    },
    {
      "epoch": 0.77,
      "learning_rate": 3.8629384468446365e-06,
      "logits/chosen": 0.5744356513023376,
      "logits/rejected": 0.5954487919807434,
      "logps/chosen": -250.98178100585938,
      "logps/rejected": -272.93701171875,
      "loss": 2119.459,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.13649822771549225,
      "rewards/margins": 0.07962769269943237,
      "rewards/rejected": -0.21612592041492462,
      "step": 1470
    },
    {
      "epoch": 0.77,
      "learning_rate": 3.84373065981799e-06,
      "logits/chosen": 0.6630114912986755,
      "logits/rejected": 0.6675506830215454,
      "logps/chosen": -265.52447509765625,
      "logps/rejected": -247.32455444335938,
      "loss": 1991.6379,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.12121538817882538,
      "rewards/margins": 0.08006526529788971,
      "rewards/rejected": -0.20128066837787628,
      "step": 1480
    },
    {
      "epoch": 0.78,
      "learning_rate": 3.824410671667948e-06,
      "logits/chosen": 0.6106497645378113,
      "logits/rejected": 0.680738091468811,
      "logps/chosen": -260.89288330078125,
      "logps/rejected": -252.52017211914062,
      "loss": 1942.9977,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.12056465446949005,
      "rewards/margins": 0.09015407413244247,
      "rewards/rejected": -0.21071875095367432,
      "step": 1490
    },
    {
      "epoch": 0.79,
      "learning_rate": 3.8049800956079552e-06,
      "logits/chosen": 0.5932056903839111,
      "logits/rejected": 0.6287232637405396,
      "logps/chosen": -291.2415771484375,
      "logps/rejected": -279.5646057128906,
      "loss": 2094.2182,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.13307559490203857,
      "rewards/margins": 0.0750352293252945,
      "rewards/rejected": -0.20811080932617188,
      "step": 1500
    },
    {
      "epoch": 0.79,
      "eval_logits/chosen": 0.5898318886756897,
      "eval_logits/rejected": 0.6474130749702454,
      "eval_logps/chosen": -270.5129089355469,
      "eval_logps/rejected": -255.21328735351562,
      "eval_loss": 2076.30419921875,
      "eval_rewards/accuracies": 0.6790000200271606,
      "eval_rewards/chosen": -0.13895468413829803,
      "eval_rewards/margins": 0.07768914848566055,
      "eval_rewards/rejected": -0.2166438102722168,
      "eval_runtime": 416.7121,
      "eval_samples_per_second": 4.799,
      "eval_steps_per_second": 1.2,
      "step": 1500
    },
    {
      "epoch": 0.79,
      "learning_rate": 3.7854405540855268e-06,
      "logits/chosen": 0.580877959728241,
      "logits/rejected": 0.6030541658401489,
      "logps/chosen": -255.82693481445312,
      "logps/rejected": -249.1620635986328,
      "loss": 2165.7623,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1440289467573166,
      "rewards/margins": 0.06519783288240433,
      "rewards/rejected": -0.2092268019914627,
      "step": 1510
    },
    {
      "epoch": 0.8,
      "learning_rate": 3.765793678646753e-06,
      "logits/chosen": 0.612065851688385,
      "logits/rejected": 0.6108434200286865,
      "logps/chosen": -236.6591796875,
      "logps/rejected": -245.4730224609375,
      "loss": 2151.6375,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.13691964745521545,
      "rewards/margins": 0.06442641466856003,
      "rewards/rejected": -0.20134606957435608,
      "step": 1520
    },
    {
      "epoch": 0.8,
      "learning_rate": 3.7460411098000804e-06,
      "logits/chosen": 0.620397686958313,
      "logits/rejected": 0.6705790758132935,
      "logps/chosen": -279.47003173828125,
      "logps/rejected": -242.50320434570312,
      "loss": 2097.6518,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.13826757669448853,
      "rewards/margins": 0.06730998307466507,
      "rewards/rejected": -0.2055775672197342,
      "step": 1530
    },
    {
      "epoch": 0.81,
      "learning_rate": 3.726184496879323e-06,
      "logits/chosen": 0.5731703042984009,
      "logits/rejected": 0.6038475036621094,
      "logps/chosen": -273.57684326171875,
      "logps/rejected": -263.6417541503906,
      "loss": 1980.56,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.13297039270401,
      "rewards/margins": 0.08892510086297989,
      "rewards/rejected": -0.2218955010175705,
      "step": 1540
    },
    {
      "epoch": 0.81,
      "learning_rate": 3.706225497905946e-06,
      "logits/chosen": 0.5495398640632629,
      "logits/rejected": 0.6184272170066833,
      "logps/chosen": -278.1634521484375,
      "logps/rejected": -250.0457763671875,
      "loss": 1925.0881,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.14073289930820465,
      "rewards/margins": 0.08853240311145782,
      "rewards/rejected": -0.22926530241966248,
      "step": 1550
    },
    {
      "epoch": 0.82,
      "learning_rate": 3.686165779450619e-06,
      "logits/chosen": 0.6478545069694519,
      "logits/rejected": 0.6362086534500122,
      "logps/chosen": -267.02618408203125,
      "logps/rejected": -239.1699676513672,
      "loss": 2063.3338,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.13111211359500885,
      "rewards/margins": 0.07986196875572205,
      "rewards/rejected": -0.2109740674495697,
      "step": 1560
    },
    {
      "epoch": 0.82,
      "learning_rate": 3.6660070164940614e-06,
      "logits/chosen": 0.6316484808921814,
      "logits/rejected": 0.686813473701477,
      "logps/chosen": -254.73837280273438,
      "logps/rejected": -249.4886016845703,
      "loss": 1958.6893,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.1321488916873932,
      "rewards/margins": 0.09855500608682632,
      "rewards/rejected": -0.23070387542247772,
      "step": 1570
    },
    {
      "epoch": 0.83,
      "learning_rate": 3.645750892287178e-06,
      "logits/chosen": 0.6227657794952393,
      "logits/rejected": 0.642948567867279,
      "logps/chosen": -254.3902587890625,
      "logps/rejected": -239.3006134033203,
      "loss": 2093.4068,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14098116755485535,
      "rewards/margins": 0.07492861151695251,
      "rewards/rejected": -0.21590976417064667,
      "step": 1580
    },
    {
      "epoch": 0.83,
      "learning_rate": 3.6253990982105114e-06,
      "logits/chosen": 0.5823426842689514,
      "logits/rejected": 0.6044851541519165,
      "logps/chosen": -282.6208801269531,
      "logps/rejected": -284.2301330566406,
      "loss": 2308.56,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1489720195531845,
      "rewards/margins": 0.050694145262241364,
      "rewards/rejected": -0.19966615736484528,
      "step": 1590
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.604953333633009e-06,
      "logits/chosen": 0.6414502859115601,
      "logits/rejected": 0.6938506960868835,
      "logps/chosen": -254.117431640625,
      "logps/rejected": -234.0909423828125,
      "loss": 2171.3457,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.12779875099658966,
      "rewards/margins": 0.06595613807439804,
      "rewards/rejected": -0.1937548816204071,
      "step": 1600
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": 0.5817673802375793,
      "eval_logits/rejected": 0.6391910910606384,
      "eval_logps/chosen": -270.3594970703125,
      "eval_logps/rejected": -255.21298217773438,
      "eval_loss": 2069.375732421875,
      "eval_rewards/accuracies": 0.6809999942779541,
      "eval_rewards/chosen": -0.137420654296875,
      "eval_rewards/margins": 0.07922003418207169,
      "eval_rewards/rejected": -0.2166406810283661,
      "eval_runtime": 416.6975,
      "eval_samples_per_second": 4.8,
      "eval_steps_per_second": 1.2,
      "step": 1600
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.5844153057701303e-06,
      "logits/chosen": 0.6806268095970154,
      "logits/rejected": 0.6613883376121521,
      "logps/chosen": -293.35455322265625,
      "logps/rejected": -249.47317504882812,
      "loss": 2235.1336,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.13428549468517303,
      "rewards/margins": 0.06987977027893066,
      "rewards/rejected": -0.2041652649641037,
      "step": 1610
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.56378672954129e-06,
      "logits/chosen": 0.5934259295463562,
      "logits/rejected": 0.6393053531646729,
      "logps/chosen": -263.8625183105469,
      "logps/rejected": -268.93646240234375,
      "loss": 2115.3395,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.12568698823451996,
      "rewards/margins": 0.06675116717815399,
      "rewards/rejected": -0.19243815541267395,
      "step": 1620
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.5430693274266694e-06,
      "logits/chosen": 0.6212111711502075,
      "logits/rejected": 0.6776979565620422,
      "logps/chosen": -265.48065185546875,
      "logps/rejected": -242.78189086914062,
      "loss": 1885.1145,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.12569081783294678,
      "rewards/margins": 0.09405811876058578,
      "rewards/rejected": -0.21974892914295197,
      "step": 1630
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.5222648293233806e-06,
      "logits/chosen": 0.5869291424751282,
      "logits/rejected": 0.614780843257904,
      "logps/chosen": -309.3984069824219,
      "logps/rejected": -281.76800537109375,
      "loss": 2295.801,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13158050179481506,
      "rewards/margins": 0.05296233296394348,
      "rewards/rejected": -0.18454284965991974,
      "step": 1640
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.5013749724010298e-06,
      "logits/chosen": 0.6291738152503967,
      "logits/rejected": 0.6847606897354126,
      "logps/chosen": -269.4018249511719,
      "logps/rejected": -248.9547576904297,
      "loss": 1985.4633,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.12610626220703125,
      "rewards/margins": 0.08754386752843857,
      "rewards/rejected": -0.21365013718605042,
      "step": 1650
    },
    {
      "epoch": 0.87,
      "learning_rate": 3.4804015009566573e-06,
      "logits/chosen": 0.6193640828132629,
      "logits/rejected": 0.6583009958267212,
      "logps/chosen": -260.786865234375,
      "logps/rejected": -246.417724609375,
      "loss": 2067.8201,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13031774759292603,
      "rewards/margins": 0.07650937139987946,
      "rewards/rejected": -0.20682711899280548,
      "step": 1660
    },
    {
      "epoch": 0.87,
      "learning_rate": 3.459346166269093e-06,
      "logits/chosen": 0.569218635559082,
      "logits/rejected": 0.6178910136222839,
      "logps/chosen": -286.010498046875,
      "logps/rejected": -284.7559509277344,
      "loss": 2057.5484,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.11479117721319199,
      "rewards/margins": 0.08598540723323822,
      "rewards/rejected": -0.20077654719352722,
      "step": 1670
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.4382107264527244e-06,
      "logits/chosen": 0.6346784234046936,
      "logits/rejected": 0.7338213920593262,
      "logps/chosen": -300.38739013671875,
      "logps/rejected": -259.44525146484375,
      "loss": 1994.4248,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.13437870144844055,
      "rewards/margins": 0.08268047124147415,
      "rewards/rejected": -0.2170591652393341,
      "step": 1680
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.416996946310694e-06,
      "logits/chosen": 0.5468164086341858,
      "logits/rejected": 0.5939579010009766,
      "logps/chosen": -299.57061767578125,
      "logps/rejected": -265.4569396972656,
      "loss": 1896.8961,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.11913790553808212,
      "rewards/margins": 0.10197613388299942,
      "rewards/rejected": -0.22111406922340393,
      "step": 1690
    },
    {
      "epoch": 0.89,
      "learning_rate": 3.3957065971875387e-06,
      "logits/chosen": 0.5587860345840454,
      "logits/rejected": 0.6276572346687317,
      "logps/chosen": -253.8291473388672,
      "logps/rejected": -239.05685424804688,
      "loss": 2189.3863,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.14831021428108215,
      "rewards/margins": 0.06738562881946564,
      "rewards/rejected": -0.2156958281993866,
      "step": 1700
    },
    {
      "epoch": 0.89,
      "eval_logits/chosen": 0.5722830295562744,
      "eval_logits/rejected": 0.629075288772583,
      "eval_logps/chosen": -270.473876953125,
      "eval_logps/rejected": -255.46749877929688,
      "eval_loss": 2062.199462890625,
      "eval_rewards/accuracies": 0.6779999732971191,
      "eval_rewards/chosen": -0.13856419920921326,
      "eval_rewards/margins": 0.08062165975570679,
      "eval_rewards/rejected": -0.21918585896492004,
      "eval_runtime": 416.4246,
      "eval_samples_per_second": 4.803,
      "eval_steps_per_second": 1.201,
      "step": 1700
    },
    {
      "epoch": 0.9,
      "learning_rate": 3.3743414568212828e-06,
      "logits/chosen": 0.6158628463745117,
      "logits/rejected": 0.6673662066459656,
      "logps/chosen": -296.7020263671875,
      "logps/rejected": -242.6101837158203,
      "loss": 2082.5406,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.13253936171531677,
      "rewards/margins": 0.07927088439464569,
      "rewards/rejected": -0.21181027591228485,
      "step": 1710
    },
    {
      "epoch": 0.9,
      "learning_rate": 3.352903309194999e-06,
      "logits/chosen": 0.6274576187133789,
      "logits/rejected": 0.6294026374816895,
      "logps/chosen": -293.9241638183594,
      "logps/rejected": -253.40036010742188,
      "loss": 2083.2709,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.13250832259655,
      "rewards/margins": 0.08279639482498169,
      "rewards/rejected": -0.21530470252037048,
      "step": 1720
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.331393944387845e-06,
      "logits/chosen": 0.5965205430984497,
      "logits/rejected": 0.70032799243927,
      "logps/chosen": -291.1014099121094,
      "logps/rejected": -274.6158752441406,
      "loss": 2108.5279,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.12708380818367004,
      "rewards/margins": 0.08121255040168762,
      "rewards/rejected": -0.20829637348651886,
      "step": 1730
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.309815158425591e-06,
      "logits/chosen": 0.588997483253479,
      "logits/rejected": 0.6009566783905029,
      "logps/chosen": -244.7908172607422,
      "logps/rejected": -238.26651000976562,
      "loss": 2017.7098,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11969755589962006,
      "rewards/margins": 0.07758014649152756,
      "rewards/rejected": -0.19727769494056702,
      "step": 1740
    },
    {
      "epoch": 0.92,
      "learning_rate": 3.288168753130657e-06,
      "logits/chosen": 0.6095719933509827,
      "logits/rejected": 0.6279308199882507,
      "logps/chosen": -250.91116333007812,
      "logps/rejected": -265.10302734375,
      "loss": 1975.5611,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1158142238855362,
      "rewards/margins": 0.08399386703968048,
      "rewards/rejected": -0.19980809092521667,
      "step": 1750
    },
    {
      "epoch": 0.92,
      "learning_rate": 3.266456535971654e-06,
      "logits/chosen": 0.5891221165657043,
      "logits/rejected": 0.5675392746925354,
      "logps/chosen": -283.29901123046875,
      "logps/rejected": -258.32562255859375,
      "loss": 1991.3586,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.1158837080001831,
      "rewards/margins": 0.09360859543085098,
      "rewards/rejected": -0.2094922959804535,
      "step": 1760
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.2446803199124666e-06,
      "logits/chosen": 0.542614221572876,
      "logits/rejected": 0.5660384893417358,
      "logps/chosen": -260.6263427734375,
      "logps/rejected": -240.3424835205078,
      "loss": 2044.5988,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.12066911160945892,
      "rewards/margins": 0.0799749344587326,
      "rewards/rejected": -0.20064406096935272,
      "step": 1770
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.2228419232608692e-06,
      "logits/chosen": 0.5963379144668579,
      "logits/rejected": 0.6299723386764526,
      "logps/chosen": -248.21920776367188,
      "logps/rejected": -235.8214874267578,
      "loss": 2095.258,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.12164044380187988,
      "rewards/margins": 0.07495652884244919,
      "rewards/rejected": -0.19659698009490967,
      "step": 1780
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.2009431695166985e-06,
      "logits/chosen": 0.5749480724334717,
      "logits/rejected": 0.627223014831543,
      "logps/chosen": -239.6404266357422,
      "logps/rejected": -239.62014770507812,
      "loss": 1970.9955,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.11313848197460175,
      "rewards/margins": 0.08247244358062744,
      "rewards/rejected": -0.19561094045639038,
      "step": 1790
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.1789858872195888e-06,
      "logits/chosen": 0.6324980854988098,
      "logits/rejected": 0.6260117888450623,
      "logps/chosen": -244.56362915039062,
      "logps/rejected": -245.474609375,
      "loss": 2292.8938,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.14712049067020416,
      "rewards/margins": 0.05482936650514603,
      "rewards/rejected": -0.2019498646259308,
      "step": 1800
    },
    {
      "epoch": 0.94,
      "eval_logits/chosen": 0.5703141689300537,
      "eval_logits/rejected": 0.627535343170166,
      "eval_logps/chosen": -268.5789489746094,
      "eval_logps/rejected": -253.6024627685547,
      "eval_loss": 2053.1298828125,
      "eval_rewards/accuracies": 0.6830000281333923,
      "eval_rewards/chosen": -0.11961515992879868,
      "eval_rewards/margins": 0.08092045783996582,
      "eval_rewards/rejected": -0.2005356103181839,
      "eval_runtime": 416.7248,
      "eval_samples_per_second": 4.799,
      "eval_steps_per_second": 1.2,
      "step": 1800
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.156971909796295e-06,
      "logits/chosen": 0.6370185613632202,
      "logits/rejected": 0.7445378303527832,
      "logps/chosen": -265.6059265136719,
      "logps/rejected": -232.7034454345703,
      "loss": 1958.5383,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.11271758377552032,
      "rewards/margins": 0.08874475955963135,
      "rewards/rejected": -0.20146234333515167,
      "step": 1810
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.1349030754075945e-06,
      "logits/chosen": 0.623261034488678,
      "logits/rejected": 0.6591364741325378,
      "logps/chosen": -263.49993896484375,
      "logps/rejected": -241.6349334716797,
      "loss": 1940.6969,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1059044599533081,
      "rewards/margins": 0.09299680590629578,
      "rewards/rejected": -0.19890126585960388,
      "step": 1820
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.1127812267948095e-06,
      "logits/chosen": 0.6355741024017334,
      "logits/rejected": 0.6655168533325195,
      "logps/chosen": -264.20062255859375,
      "logps/rejected": -258.43310546875,
      "loss": 1993.0014,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.12248332798480988,
      "rewards/margins": 0.08727528899908066,
      "rewards/rejected": -0.20975859463214874,
      "step": 1830
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.0906082111259313e-06,
      "logits/chosen": 0.548941433429718,
      "logits/rejected": 0.5715293884277344,
      "logps/chosen": -277.3153076171875,
      "logps/rejected": -248.1029510498047,
      "loss": 2108.9805,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.10435888916254044,
      "rewards/margins": 0.07835109531879425,
      "rewards/rejected": -0.18270999193191528,
      "step": 1840
    },
    {
      "epoch": 0.97,
      "learning_rate": 3.068385879841389e-06,
      "logits/chosen": 0.6165980100631714,
      "logits/rejected": 0.6937299966812134,
      "logps/chosen": -233.5325469970703,
      "logps/rejected": -246.7562255859375,
      "loss": 2168.8945,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10994801670312881,
      "rewards/margins": 0.06389383226633072,
      "rewards/rejected": -0.17384183406829834,
      "step": 1850
    },
    {
      "epoch": 0.97,
      "learning_rate": 3.046116088499449e-06,
      "logits/chosen": 0.6379483938217163,
      "logits/rejected": 0.6270259618759155,
      "logps/chosen": -266.16009521484375,
      "logps/rejected": -259.60919189453125,
      "loss": 2145.4496,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.11401374638080597,
      "rewards/margins": 0.07185572385787964,
      "rewards/rejected": -0.1858694702386856,
      "step": 1860
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.02380069662128e-06,
      "logits/chosen": 0.623966634273529,
      "logits/rejected": 0.5938777327537537,
      "logps/chosen": -252.69869995117188,
      "logps/rejected": -245.094482421875,
      "loss": 2034.7914,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.10203299671411514,
      "rewards/margins": 0.07526172697544098,
      "rewards/rejected": -0.17729471623897552,
      "step": 1870
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.0014415675356813e-06,
      "logits/chosen": 0.6284725069999695,
      "logits/rejected": 0.6297743916511536,
      "logps/chosen": -270.23333740234375,
      "logps/rejected": -252.12704467773438,
      "loss": 2150.0496,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10082075744867325,
      "rewards/margins": 0.06684517115354538,
      "rewards/rejected": -0.16766592860221863,
      "step": 1880
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.979040568223498e-06,
      "logits/chosen": 0.5534299612045288,
      "logits/rejected": 0.675399661064148,
      "logps/chosen": -263.3745422363281,
      "logps/rejected": -266.8883972167969,
      "loss": 2205.4939,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.11857882887125015,
      "rewards/margins": 0.06667140126228333,
      "rewards/rejected": -0.18525022268295288,
      "step": 1890
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.9565995691617242e-06,
      "logits/chosen": 0.6073340773582458,
      "logits/rejected": 0.6487486362457275,
      "logps/chosen": -292.90704345703125,
      "logps/rejected": -238.2965545654297,
      "loss": 2085.5805,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.09591428190469742,
      "rewards/margins": 0.08266115188598633,
      "rewards/rejected": -0.17857542634010315,
      "step": 1900
    },
    {
      "epoch": 0.99,
      "eval_logits/chosen": 0.5747328996658325,
      "eval_logits/rejected": 0.6318737864494324,
      "eval_logps/chosen": -267.4730224609375,
      "eval_logps/rejected": -252.61306762695312,
      "eval_loss": 2052.32373046875,
      "eval_rewards/accuracies": 0.6899999976158142,
      "eval_rewards/chosen": -0.10855603218078613,
      "eval_rewards/margins": 0.08208546042442322,
      "eval_rewards/rejected": -0.19064147770404816,
      "eval_runtime": 416.81,
      "eval_samples_per_second": 4.798,
      "eval_steps_per_second": 1.2,
      "step": 1900
    },
    {
      "epoch": 1.0,
      "learning_rate": 2.9341204441673267e-06,
      "logits/chosen": 0.5886205434799194,
      "logits/rejected": 0.6063315272331238,
      "logps/chosen": -281.51129150390625,
      "logps/rejected": -263.0057678222656,
      "loss": 1992.2906,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10820697247982025,
      "rewards/margins": 0.08012684434652328,
      "rewards/rejected": -0.18833380937576294,
      "step": 1910
    },
    {
      "epoch": 1.0,
      "learning_rate": 2.9116050702407706e-06,
      "logits/chosen": 0.6380060315132141,
      "logits/rejected": 0.6841104030609131,
      "logps/chosen": -267.55145263671875,
      "logps/rejected": -248.828369140625,
      "loss": 2045.3801,
      "rewards/accuracies": 0.6604167222976685,
      "rewards/chosen": -0.10555033385753632,
      "rewards/margins": 0.08623509109020233,
      "rewards/rejected": -0.19178542494773865,
      "step": 1920
    },
    {
      "epoch": 1.01,
      "learning_rate": 2.889055327409301e-06,
      "logits/chosen": 0.5285671353340149,
      "logits/rejected": 0.5704804062843323,
      "logps/chosen": -263.24725341796875,
      "logps/rejected": -248.02395629882812,
      "loss": 2023.134,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.10273389518260956,
      "rewards/margins": 0.08054333180189133,
      "rewards/rejected": -0.1832772046327591,
      "step": 1930
    },
    {
      "epoch": 1.02,
      "learning_rate": 2.8664730985699537e-06,
      "logits/chosen": 0.5331718325614929,
      "logits/rejected": 0.5988043546676636,
      "logps/chosen": -242.79061889648438,
      "logps/rejected": -238.0269012451172,
      "loss": 1960.2693,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09900447726249695,
      "rewards/margins": 0.0868852287530899,
      "rewards/rejected": -0.18588972091674805,
      "step": 1940
    },
    {
      "epoch": 1.02,
      "learning_rate": 2.843860269332339e-06,
      "logits/chosen": 0.6072074174880981,
      "logits/rejected": 0.631058394908905,
      "logps/chosen": -273.4151306152344,
      "logps/rejected": -246.41238403320312,
      "loss": 1955.907,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09934862703084946,
      "rewards/margins": 0.09240168333053589,
      "rewards/rejected": -0.19175033271312714,
      "step": 1950
    },
    {
      "epoch": 1.03,
      "learning_rate": 2.8212187278611907e-06,
      "logits/chosen": 0.6683967113494873,
      "logits/rejected": 0.6856907606124878,
      "logps/chosen": -257.2086181640625,
      "logps/rejected": -247.91683959960938,
      "loss": 1924.8703,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1071348562836647,
      "rewards/margins": 0.09893598407506943,
      "rewards/rejected": -0.20607082545757294,
      "step": 1960
    },
    {
      "epoch": 1.03,
      "learning_rate": 2.7985503647187063e-06,
      "logits/chosen": 0.5825555920600891,
      "logits/rejected": 0.6476297378540039,
      "logps/chosen": -288.1867980957031,
      "logps/rejected": -254.5727996826172,
      "loss": 1907.2662,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.08619461953639984,
      "rewards/margins": 0.10769355297088623,
      "rewards/rejected": -0.19388815760612488,
      "step": 1970
    },
    {
      "epoch": 1.04,
      "learning_rate": 2.7758570727066843e-06,
      "logits/chosen": 0.5205335021018982,
      "logits/rejected": 0.6433119177818298,
      "logps/chosen": -261.37982177734375,
      "logps/rejected": -240.10952758789062,
      "loss": 2241.4863,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.12351379543542862,
      "rewards/margins": 0.05891970917582512,
      "rewards/rejected": -0.18243351578712463,
      "step": 1980
    },
    {
      "epoch": 1.04,
      "learning_rate": 2.753140746708477e-06,
      "logits/chosen": 0.6216637492179871,
      "logits/rejected": 0.669810950756073,
      "logps/chosen": -282.500244140625,
      "logps/rejected": -273.12310791015625,
      "loss": 1961.7119,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.09472217410802841,
      "rewards/margins": 0.09439438581466675,
      "rewards/rejected": -0.18911656737327576,
      "step": 1990
    },
    {
      "epoch": 1.05,
      "learning_rate": 2.730403283530767e-06,
      "logits/chosen": 0.638060986995697,
      "logits/rejected": 0.7034865617752075,
      "logps/chosen": -258.02447509765625,
      "logps/rejected": -246.9062957763672,
      "loss": 1847.759,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09892908483743668,
      "rewards/margins": 0.10315445810556412,
      "rewards/rejected": -0.2020835429430008,
      "step": 2000
    },
    {
      "epoch": 1.05,
      "eval_logits/chosen": 0.5763067603111267,
      "eval_logits/rejected": 0.6332587599754333,
      "eval_logps/chosen": -267.7949523925781,
      "eval_logps/rejected": -253.0826873779297,
      "eval_loss": 2050.417724609375,
      "eval_rewards/accuracies": 0.6850000023841858,
      "eval_rewards/chosen": -0.11177488416433334,
      "eval_rewards/margins": 0.0835629403591156,
      "eval_rewards/rejected": -0.19533783197402954,
      "eval_runtime": 416.585,
      "eval_samples_per_second": 4.801,
      "eval_steps_per_second": 1.2,
      "step": 2000
    },
    {
      "epoch": 1.05,
      "learning_rate": 2.707646581745188e-06,
      "logits/chosen": 0.6024230718612671,
      "logits/rejected": 0.6550949811935425,
      "logps/chosen": -275.99761962890625,
      "logps/rejected": -272.4120178222656,
      "loss": 1981.8529,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.0975821316242218,
      "rewards/margins": 0.08785782754421234,
      "rewards/rejected": -0.18543997406959534,
      "step": 2010
    },
    {
      "epoch": 1.06,
      "learning_rate": 2.6848725415297888e-06,
      "logits/chosen": 0.629960298538208,
      "logits/rejected": 0.6249616742134094,
      "logps/chosen": -256.87603759765625,
      "logps/rejected": -253.21109008789062,
      "loss": 2136.5217,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.10265711694955826,
      "rewards/margins": 0.0642293393611908,
      "rewards/rejected": -0.16688646376132965,
      "step": 2020
    },
    {
      "epoch": 1.06,
      "learning_rate": 2.6620830645103753e-06,
      "logits/chosen": 0.6109344959259033,
      "logits/rejected": 0.6072026491165161,
      "logps/chosen": -266.4075012207031,
      "logps/rejected": -258.20208740234375,
      "loss": 1938.8361,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.09916529804468155,
      "rewards/margins": 0.09021677076816559,
      "rewards/rejected": -0.18938204646110535,
      "step": 2030
    },
    {
      "epoch": 1.07,
      "learning_rate": 2.639280053601719e-06,
      "logits/chosen": 0.566746711730957,
      "logits/rejected": 0.6063026189804077,
      "logps/chosen": -261.76739501953125,
      "logps/rejected": -270.6283874511719,
      "loss": 2085.5938,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.12018795311450958,
      "rewards/margins": 0.08117975294589996,
      "rewards/rejected": -0.20136770606040955,
      "step": 2040
    },
    {
      "epoch": 1.07,
      "learning_rate": 2.6164654128486683e-06,
      "logits/chosen": 0.5058253407478333,
      "logits/rejected": 0.6028685569763184,
      "logps/chosen": -267.67376708984375,
      "logps/rejected": -230.2966766357422,
      "loss": 2055.1498,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11003967374563217,
      "rewards/margins": 0.08447955548763275,
      "rewards/rejected": -0.19451923668384552,
      "step": 2050
    },
    {
      "epoch": 1.08,
      "learning_rate": 2.59364104726716e-06,
      "logits/chosen": 0.5947778820991516,
      "logits/rejected": 0.5901384353637695,
      "logps/chosen": -278.0001525878906,
      "logps/rejected": -242.13583374023438,
      "loss": 1877.0174,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11745290458202362,
      "rewards/margins": 0.10013137012720108,
      "rewards/rejected": -0.2175842821598053,
      "step": 2060
    },
    {
      "epoch": 1.08,
      "learning_rate": 2.5708088626851546e-06,
      "logits/chosen": 0.5502884387969971,
      "logits/rejected": 0.603992760181427,
      "logps/chosen": -269.38360595703125,
      "logps/rejected": -244.87619018554688,
      "loss": 2015.0283,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.12011172622442245,
      "rewards/margins": 0.08565986156463623,
      "rewards/rejected": -0.20577159523963928,
      "step": 2070
    },
    {
      "epoch": 1.09,
      "learning_rate": 2.547970765583491e-06,
      "logits/chosen": 0.5619412064552307,
      "logits/rejected": 0.6468341946601868,
      "logps/chosen": -252.68115234375,
      "logps/rejected": -252.705322265625,
      "loss": 1855.9893,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10235454142093658,
      "rewards/margins": 0.10716482251882553,
      "rewards/rejected": -0.20951935648918152,
      "step": 2080
    },
    {
      "epoch": 1.09,
      "learning_rate": 2.525128662936707e-06,
      "logits/chosen": 0.512058436870575,
      "logits/rejected": 0.5677643418312073,
      "logps/chosen": -270.7825012207031,
      "logps/rejected": -260.822509765625,
      "loss": 1831.3346,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.1064097136259079,
      "rewards/margins": 0.10622493177652359,
      "rewards/rejected": -0.21263465285301208,
      "step": 2090
    },
    {
      "epoch": 1.1,
      "learning_rate": 2.502284462053799e-06,
      "logits/chosen": 0.620409369468689,
      "logits/rejected": 0.6358670592308044,
      "logps/chosen": -258.42706298828125,
      "logps/rejected": -258.72161865234375,
      "loss": 2024.9559,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10622759163379669,
      "rewards/margins": 0.08639432489871979,
      "rewards/rejected": -0.19262190163135529,
      "step": 2100
    },
    {
      "epoch": 1.1,
      "eval_logits/chosen": 0.5589507818222046,
      "eval_logits/rejected": 0.6156801581382751,
      "eval_logps/chosen": -268.8072814941406,
      "eval_logps/rejected": -254.37991333007812,
      "eval_loss": 2046.75927734375,
      "eval_rewards/accuracies": 0.6899999976158142,
      "eval_rewards/chosen": -0.12189868092536926,
      "eval_rewards/margins": 0.08641137927770615,
      "eval_rewards/rejected": -0.2083100527524948,
      "eval_runtime": 416.7259,
      "eval_samples_per_second": 4.799,
      "eval_steps_per_second": 1.2,
      "step": 2100
    },
    {
      "epoch": 1.1,
      "learning_rate": 2.479440070418967e-06,
      "logits/chosen": 0.5901846885681152,
      "logits/rejected": 0.6195170283317566,
      "logps/chosen": -249.45816040039062,
      "logps/rejected": -253.7944793701172,
      "loss": 2205.6043,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.13660283386707306,
      "rewards/margins": 0.06599629670381546,
      "rewards/rejected": -0.20259912312030792,
      "step": 2110
    },
    {
      "epoch": 1.11,
      "learning_rate": 2.456597395532338e-06,
      "logits/chosen": 0.5504690408706665,
      "logits/rejected": 0.6531665921211243,
      "logps/chosen": -259.79010009765625,
      "logps/rejected": -284.0751647949219,
      "loss": 1959.0818,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.11810295283794403,
      "rewards/margins": 0.0964241549372673,
      "rewards/rejected": -0.21452713012695312,
      "step": 2120
    },
    {
      "epoch": 1.11,
      "learning_rate": 2.433758344750691e-06,
      "logits/chosen": 0.5741318464279175,
      "logits/rejected": 0.6458116173744202,
      "logps/chosen": -295.03192138671875,
      "logps/rejected": -276.64251708984375,
      "loss": 1911.3146,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.11703141778707504,
      "rewards/margins": 0.10501817613840103,
      "rewards/rejected": -0.2220495641231537,
      "step": 2130
    },
    {
      "epoch": 1.12,
      "learning_rate": 2.4109248251281953e-06,
      "logits/chosen": 0.5908122062683105,
      "logits/rejected": 0.6558480858802795,
      "logps/chosen": -283.0213928222656,
      "logps/rejected": -252.56600952148438,
      "loss": 1911.8793,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11346153914928436,
      "rewards/margins": 0.10086224228143692,
      "rewards/rejected": -0.2143237590789795,
      "step": 2140
    },
    {
      "epoch": 1.13,
      "learning_rate": 2.3880987432571675e-06,
      "logits/chosen": 0.5616129040718079,
      "logits/rejected": 0.593204915523529,
      "logps/chosen": -268.16583251953125,
      "logps/rejected": -262.17755126953125,
      "loss": 1994.0697,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.11974780261516571,
      "rewards/margins": 0.09457085281610489,
      "rewards/rejected": -0.2143186628818512,
      "step": 2150
    },
    {
      "epoch": 1.13,
      "learning_rate": 2.365282005108875e-06,
      "logits/chosen": 0.5762392282485962,
      "logits/rejected": 0.615722119808197,
      "logps/chosen": -250.62509155273438,
      "logps/rejected": -253.91049194335938,
      "loss": 2099.1633,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.1276218444108963,
      "rewards/margins": 0.07122951745986938,
      "rewards/rejected": -0.19885137677192688,
      "step": 2160
    },
    {
      "epoch": 1.14,
      "learning_rate": 2.3424765158743867e-06,
      "logits/chosen": 0.6059794425964355,
      "logits/rejected": 0.6645799875259399,
      "logps/chosen": -255.7693634033203,
      "logps/rejected": -251.96951293945312,
      "loss": 2010.217,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12624487280845642,
      "rewards/margins": 0.09581606835126877,
      "rewards/rejected": -0.2220609486103058,
      "step": 2170
    },
    {
      "epoch": 1.14,
      "learning_rate": 2.319684179805491e-06,
      "logits/chosen": 0.516992449760437,
      "logits/rejected": 0.5456847548484802,
      "logps/chosen": -265.10321044921875,
      "logps/rejected": -246.3701629638672,
      "loss": 1933.4984,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.11627723276615143,
      "rewards/margins": 0.0940864160656929,
      "rewards/rejected": -0.21036362648010254,
      "step": 2180
    },
    {
      "epoch": 1.15,
      "learning_rate": 2.296906900055691e-06,
      "logits/chosen": 0.596808135509491,
      "logits/rejected": 0.6393652558326721,
      "logps/chosen": -264.455810546875,
      "logps/rejected": -256.40667724609375,
      "loss": 2172.6984,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.12352782487869263,
      "rewards/margins": 0.07605434954166412,
      "rewards/rejected": -0.19958215951919556,
      "step": 2190
    },
    {
      "epoch": 1.15,
      "learning_rate": 2.2741465785212905e-06,
      "logits/chosen": 0.5940336585044861,
      "logits/rejected": 0.6305769085884094,
      "logps/chosen": -256.6434326171875,
      "logps/rejected": -245.391357421875,
      "loss": 2038.6354,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11867289245128632,
      "rewards/margins": 0.09147666394710541,
      "rewards/rejected": -0.21014957129955292,
      "step": 2200
    },
    {
      "epoch": 1.15,
      "eval_logits/chosen": 0.5517618656158447,
      "eval_logits/rejected": 0.6082795858383179,
      "eval_logps/chosen": -268.6722106933594,
      "eval_logps/rejected": -254.27310180664062,
      "eval_loss": 2043.57275390625,
      "eval_rewards/accuracies": 0.6880000233650208,
      "eval_rewards/chosen": -0.12054779380559921,
      "eval_rewards/margins": 0.08669425547122955,
      "eval_rewards/rejected": -0.20724207162857056,
      "eval_runtime": 416.7766,
      "eval_samples_per_second": 4.799,
      "eval_steps_per_second": 1.2,
      "step": 2200
    },
    {
      "epoch": 1.16,
      "learning_rate": 2.251405115682587e-06,
      "logits/chosen": 0.5902246236801147,
      "logits/rejected": 0.5983418822288513,
      "logps/chosen": -263.2071228027344,
      "logps/rejected": -272.0802307128906,
      "loss": 2017.8775,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.11642640829086304,
      "rewards/margins": 0.09181423485279083,
      "rewards/rejected": -0.20824062824249268,
      "step": 2210
    },
    {
      "epoch": 1.16,
      "learning_rate": 2.2286844104451848e-06,
      "logits/chosen": 0.5431746244430542,
      "logits/rejected": 0.6418278217315674,
      "logps/chosen": -264.33465576171875,
      "logps/rejected": -251.9208984375,
      "loss": 2095.6342,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.10533900558948517,
      "rewards/margins": 0.07776842266321182,
      "rewards/rejected": -0.1831074208021164,
      "step": 2220
    },
    {
      "epoch": 1.17,
      "learning_rate": 2.205986359981431e-06,
      "logits/chosen": 0.5207514762878418,
      "logits/rejected": 0.6270573139190674,
      "logps/chosen": -285.76849365234375,
      "logps/rejected": -277.93426513671875,
      "loss": 1912.0979,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.11343447118997574,
      "rewards/margins": 0.1047770231962204,
      "rewards/rejected": -0.21821150183677673,
      "step": 2230
    },
    {
      "epoch": 1.17,
      "learning_rate": 2.183312859572008e-06,
      "logits/chosen": 0.5806037783622742,
      "logits/rejected": 0.6543610095977783,
      "logps/chosen": -281.1925354003906,
      "logps/rejected": -278.2233581542969,
      "loss": 2272.857,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.11976996809244156,
      "rewards/margins": 0.06387045979499817,
      "rewards/rejected": -0.18364043533802032,
      "step": 2240
    },
    {
      "epoch": 1.18,
      "learning_rate": 2.1606658024476744e-06,
      "logits/chosen": 0.5554038286209106,
      "logits/rejected": 0.5429580211639404,
      "logps/chosen": -269.9796447753906,
      "logps/rejected": -250.5005340576172,
      "loss": 2188.1607,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1146581768989563,
      "rewards/margins": 0.07278671860694885,
      "rewards/rejected": -0.18744489550590515,
      "step": 2250
    },
    {
      "epoch": 1.18,
      "learning_rate": 2.1380470796311843e-06,
      "logits/chosen": 0.610127866268158,
      "logits/rejected": 0.6246207356452942,
      "logps/chosen": -274.421142578125,
      "logps/rejected": -259.88336181640625,
      "loss": 1878.2621,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.10982956737279892,
      "rewards/margins": 0.10056765377521515,
      "rewards/rejected": -0.21039721369743347,
      "step": 2260
    },
    {
      "epoch": 1.19,
      "learning_rate": 2.1154585797793826e-06,
      "logits/chosen": 0.6410681009292603,
      "logits/rejected": 0.6446506977081299,
      "logps/chosen": -262.7099304199219,
      "logps/rejected": -243.33847045898438,
      "loss": 1969.0021,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.10027629137039185,
      "rewards/margins": 0.08689162135124207,
      "rewards/rejected": -0.1871679127216339,
      "step": 2270
    },
    {
      "epoch": 1.19,
      "learning_rate": 2.092902189025507e-06,
      "logits/chosen": 0.6191312670707703,
      "logits/rejected": 0.6812275648117065,
      "logps/chosen": -258.3789978027344,
      "logps/rejected": -247.3417510986328,
      "loss": 1765.025,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10864460468292236,
      "rewards/margins": 0.11250102519989014,
      "rewards/rejected": -0.2211456298828125,
      "step": 2280
    },
    {
      "epoch": 1.2,
      "learning_rate": 2.070379790821693e-06,
      "logits/chosen": 0.5654376745223999,
      "logits/rejected": 0.6462022066116333,
      "logps/chosen": -301.2412414550781,
      "logps/rejected": -276.8460998535156,
      "loss": 2045.4492,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10371474176645279,
      "rewards/margins": 0.09003494679927826,
      "rewards/rejected": -0.19374969601631165,
      "step": 2290
    },
    {
      "epoch": 1.2,
      "learning_rate": 2.0478932657817105e-06,
      "logits/chosen": 0.5810787081718445,
      "logits/rejected": 0.6360457539558411,
      "logps/chosen": -254.06838989257812,
      "logps/rejected": -243.23989868164062,
      "loss": 2022.9617,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11232425272464752,
      "rewards/margins": 0.07984773069620132,
      "rewards/rejected": -0.19217197597026825,
      "step": 2300
    },
    {
      "epoch": 1.2,
      "eval_logits/chosen": 0.5535383820533752,
      "eval_logits/rejected": 0.6101322174072266,
      "eval_logps/chosen": -268.3490905761719,
      "eval_logps/rejected": -253.95965576171875,
      "eval_loss": 2035.585693359375,
      "eval_rewards/accuracies": 0.6894999742507935,
      "eval_rewards/chosen": -0.11731643229722977,
      "eval_rewards/margins": 0.08679118007421494,
      "eval_rewards/rejected": -0.2041076123714447,
      "eval_runtime": 416.4094,
      "eval_samples_per_second": 4.803,
      "eval_steps_per_second": 1.201,
      "step": 2300
    },
    {
      "epoch": 1.21,
      "learning_rate": 2.0254444915239287e-06,
      "logits/chosen": 0.5468884706497192,
      "logits/rejected": 0.5753307938575745,
      "logps/chosen": -271.94940185546875,
      "logps/rejected": -244.8318328857422,
      "loss": 1994.7408,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.1074294000864029,
      "rewards/margins": 0.08415937423706055,
      "rewards/rejected": -0.19158877432346344,
      "step": 2310
    },
    {
      "epoch": 1.21,
      "learning_rate": 2.0030353425145376e-06,
      "logits/chosen": 0.6623051762580872,
      "logits/rejected": 0.6782322525978088,
      "logps/chosen": -220.7380828857422,
      "logps/rejected": -242.05850219726562,
      "loss": 1864.7361,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.10245666652917862,
      "rewards/margins": 0.10949740558862686,
      "rewards/rejected": -0.2119540423154831,
      "step": 2320
    },
    {
      "epoch": 1.22,
      "learning_rate": 1.9806676899110305e-06,
      "logits/chosen": 0.6308891773223877,
      "logits/rejected": 0.6477428674697876,
      "logps/chosen": -262.88897705078125,
      "logps/rejected": -255.49362182617188,
      "loss": 1843.8875,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.10279623419046402,
      "rewards/margins": 0.11021213233470917,
      "rewards/rejected": -0.21300837397575378,
      "step": 2330
    },
    {
      "epoch": 1.22,
      "learning_rate": 1.958343401405964e-06,
      "logits/chosen": 0.5211482048034668,
      "logits/rejected": 0.6105703115463257,
      "logps/chosen": -272.09698486328125,
      "logps/rejected": -240.05392456054688,
      "loss": 1991.6119,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.12355583906173706,
      "rewards/margins": 0.08727772533893585,
      "rewards/rejected": -0.2108335793018341,
      "step": 2340
    },
    {
      "epoch": 1.23,
      "learning_rate": 1.9360643410710027e-06,
      "logits/chosen": 0.6230972409248352,
      "logits/rejected": 0.6428076028823853,
      "logps/chosen": -297.76300048828125,
      "logps/rejected": -262.3421325683594,
      "loss": 2047.1437,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.10469541698694229,
      "rewards/margins": 0.09010159224271774,
      "rewards/rejected": -0.19479700922966003,
      "step": 2350
    },
    {
      "epoch": 1.24,
      "learning_rate": 1.9138323692012734e-06,
      "logits/chosen": 0.5903237462043762,
      "logits/rejected": 0.6455060243606567,
      "logps/chosen": -288.16815185546875,
      "logps/rejected": -289.3240966796875,
      "loss": 1579.4779,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.08982647955417633,
      "rewards/margins": 0.14532434940338135,
      "rewards/rejected": -0.23515084385871887,
      "step": 2360
    },
    {
      "epoch": 1.24,
      "learning_rate": 1.8916493421600287e-06,
      "logits/chosen": 0.5603612065315247,
      "logits/rejected": 0.5792626142501831,
      "logps/chosen": -243.1230010986328,
      "logps/rejected": -257.9002685546875,
      "loss": 2017.2086,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.12239034473896027,
      "rewards/margins": 0.0815977230668068,
      "rewards/rejected": -0.20398807525634766,
      "step": 2370
    },
    {
      "epoch": 1.25,
      "learning_rate": 1.8695171122236443e-06,
      "logits/chosen": 0.49118170142173767,
      "logits/rejected": 0.5506534576416016,
      "logps/chosen": -268.86822509765625,
      "logps/rejected": -274.64202880859375,
      "loss": 1977.4205,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.11895406246185303,
      "rewards/margins": 0.10293842852115631,
      "rewards/rejected": -0.22189247608184814,
      "step": 2380
    },
    {
      "epoch": 1.25,
      "learning_rate": 1.84743752742695e-06,
      "logits/chosen": 0.6215322613716125,
      "logits/rejected": 0.6151038408279419,
      "logps/chosen": -265.5515441894531,
      "logps/rejected": -278.13177490234375,
      "loss": 1889.5199,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.10073964297771454,
      "rewards/margins": 0.11166242510080338,
      "rewards/rejected": -0.2124020755290985,
      "step": 2390
    },
    {
      "epoch": 1.26,
      "learning_rate": 1.8254124314089225e-06,
      "logits/chosen": 0.6138418912887573,
      "logits/rejected": 0.6189366579055786,
      "logps/chosen": -263.2386169433594,
      "logps/rejected": -239.09963989257812,
      "loss": 1871.641,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.12337759882211685,
      "rewards/margins": 0.09867466986179352,
      "rewards/rejected": -0.22205229103565216,
      "step": 2400
    },
    {
      "epoch": 1.26,
      "eval_logits/chosen": 0.5482152104377747,
      "eval_logits/rejected": 0.6045916676521301,
      "eval_logps/chosen": -268.51605224609375,
      "eval_logps/rejected": -254.28311157226562,
      "eval_loss": 2036.3372802734375,
      "eval_rewards/accuracies": 0.6894999742507935,
      "eval_rewards/chosen": -0.11898616701364517,
      "eval_rewards/margins": 0.0883559137582779,
      "eval_rewards/rejected": -0.20734207332134247,
      "eval_runtime": 416.6781,
      "eval_samples_per_second": 4.8,
      "eval_steps_per_second": 1.2,
      "step": 2400
    },
    {
      "epoch": 1.26,
      "learning_rate": 1.8034436632587394e-06,
      "logits/chosen": 0.5728852152824402,
      "logits/rejected": 0.6265703439712524,
      "logps/chosen": -237.0697784423828,
      "logps/rejected": -242.29367065429688,
      "loss": 1969.1203,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.10528033971786499,
      "rewards/margins": 0.0880698561668396,
      "rewards/rejected": -0.1933501809835434,
      "step": 2410
    },
    {
      "epoch": 1.27,
      "learning_rate": 1.781533057362221e-06,
      "logits/chosen": 0.5749053359031677,
      "logits/rejected": 0.6042163372039795,
      "logps/chosen": -278.8114013671875,
      "logps/rejected": -279.18695068359375,
      "loss": 1906.6891,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1059660091996193,
      "rewards/margins": 0.10437663644552231,
      "rewards/rejected": -0.210342675447464,
      "step": 2420
    },
    {
      "epoch": 1.27,
      "learning_rate": 1.7596824432486537e-06,
      "logits/chosen": 0.5984959602355957,
      "logits/rejected": 0.6386197209358215,
      "logps/chosen": -292.53143310546875,
      "logps/rejected": -256.42620849609375,
      "loss": 2003.0641,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.10370471328496933,
      "rewards/margins": 0.09288345277309418,
      "rewards/rejected": -0.1965881586074829,
      "step": 2430
    },
    {
      "epoch": 1.28,
      "learning_rate": 1.7378936454380277e-06,
      "logits/chosen": 0.5537322163581848,
      "logits/rejected": 0.5942158102989197,
      "logps/chosen": -246.1141815185547,
      "logps/rejected": -253.85617065429688,
      "loss": 2137.652,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13521219789981842,
      "rewards/margins": 0.0802813172340393,
      "rewards/rejected": -0.21549351513385773,
      "step": 2440
    },
    {
      "epoch": 1.28,
      "learning_rate": 1.7161684832886893e-06,
      "logits/chosen": 0.5406220555305481,
      "logits/rejected": 0.540827751159668,
      "logps/chosen": -242.9103546142578,
      "logps/rejected": -247.41921997070312,
      "loss": 2005.6266,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.12078257650136948,
      "rewards/margins": 0.09046939015388489,
      "rewards/rejected": -0.21125197410583496,
      "step": 2450
    },
    {
      "epoch": 1.29,
      "learning_rate": 1.6945087708454273e-06,
      "logits/chosen": 0.5730911493301392,
      "logits/rejected": 0.5966663956642151,
      "logps/chosen": -276.0887145996094,
      "logps/rejected": -264.39910888671875,
      "loss": 2211.1135,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12395117431879044,
      "rewards/margins": 0.07060922682285309,
      "rewards/rejected": -0.19456037878990173,
      "step": 2460
    },
    {
      "epoch": 1.29,
      "learning_rate": 1.6729163166879964e-06,
      "logits/chosen": 0.5936635136604309,
      "logits/rejected": 0.6355383396148682,
      "logps/chosen": -258.3261413574219,
      "logps/rejected": -233.99075317382812,
      "loss": 1757.0482,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1076178103685379,
      "rewards/margins": 0.11206640303134918,
      "rewards/rejected": -0.21968421339988708,
      "step": 2470
    },
    {
      "epoch": 1.3,
      "learning_rate": 1.651392923780105e-06,
      "logits/chosen": 0.6025252342224121,
      "logits/rejected": 0.6687902808189392,
      "logps/chosen": -254.2070770263672,
      "logps/rejected": -237.63967895507812,
      "loss": 2055.3113,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1099240779876709,
      "rewards/margins": 0.09202177077531815,
      "rewards/rejected": -0.20194585621356964,
      "step": 2480
    },
    {
      "epoch": 1.3,
      "learning_rate": 1.629940389318867e-06,
      "logits/chosen": 0.5291022062301636,
      "logits/rejected": 0.616036593914032,
      "logps/chosen": -294.7336730957031,
      "logps/rejected": -240.91796875,
      "loss": 1902.4217,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10780209302902222,
      "rewards/margins": 0.10337891429662704,
      "rewards/rejected": -0.21118099987506866,
      "step": 2490
    },
    {
      "epoch": 1.31,
      "learning_rate": 1.608560504584737e-06,
      "logits/chosen": 0.5608310103416443,
      "logits/rejected": 0.6271076798439026,
      "logps/chosen": -256.45770263671875,
      "logps/rejected": -253.8623809814453,
      "loss": 1907.3463,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10689397901296616,
      "rewards/margins": 0.10536620765924454,
      "rewards/rejected": -0.2122601717710495,
      "step": 2500
    },
    {
      "epoch": 1.31,
      "eval_logits/chosen": 0.5460030436515808,
      "eval_logits/rejected": 0.6022311449050903,
      "eval_logps/chosen": -268.7764587402344,
      "eval_logps/rejected": -254.62974548339844,
      "eval_loss": 2034.7010498046875,
      "eval_rewards/accuracies": 0.6880000233650208,
      "eval_rewards/chosen": -0.12159038335084915,
      "eval_rewards/margins": 0.08921793848276138,
      "eval_rewards/rejected": -0.21080833673477173,
      "eval_runtime": 416.6626,
      "eval_samples_per_second": 4.8,
      "eval_steps_per_second": 1.2,
      "step": 2500
    },
    {
      "epoch": 1.31,
      "learning_rate": 1.587255054791937e-06,
      "logits/chosen": 0.5321905016899109,
      "logits/rejected": 0.589474081993103,
      "logps/chosen": -281.2105407714844,
      "logps/rejected": -264.56298828125,
      "loss": 2016.9854,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.10593251138925552,
      "rewards/margins": 0.08659417182207108,
      "rewards/rejected": -0.1925266534090042,
      "step": 2510
    },
    {
      "epoch": 1.32,
      "learning_rate": 1.5660258189393945e-06,
      "logits/chosen": 0.5880864262580872,
      "logits/rejected": 0.6149991750717163,
      "logps/chosen": -251.75973510742188,
      "logps/rejected": -262.3134765625,
      "loss": 2130.8975,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12219718843698502,
      "rewards/margins": 0.08161304891109467,
      "rewards/rejected": -0.2038102149963379,
      "step": 2520
    },
    {
      "epoch": 1.32,
      "learning_rate": 1.5448745696621915e-06,
      "logits/chosen": 0.5654980540275574,
      "logits/rejected": 0.6478559970855713,
      "logps/chosen": -272.79864501953125,
      "logps/rejected": -258.56402587890625,
      "loss": 2114.0654,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.11762849986553192,
      "rewards/margins": 0.08065593242645264,
      "rewards/rejected": -0.19828443229198456,
      "step": 2530
    },
    {
      "epoch": 1.33,
      "learning_rate": 1.5238030730835578e-06,
      "logits/chosen": 0.5662246942520142,
      "logits/rejected": 0.6270356178283691,
      "logps/chosen": -272.17449951171875,
      "logps/rejected": -237.2474365234375,
      "loss": 2106.9717,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11701546609401703,
      "rewards/margins": 0.0775529146194458,
      "rewards/rejected": -0.19456836581230164,
      "step": 2540
    },
    {
      "epoch": 1.33,
      "learning_rate": 1.5028130886673936e-06,
      "logits/chosen": 0.5928006172180176,
      "logits/rejected": 0.641442060470581,
      "logps/chosen": -263.0660705566406,
      "logps/rejected": -252.6539306640625,
      "loss": 2000.1971,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.111760713160038,
      "rewards/margins": 0.0842764601111412,
      "rewards/rejected": -0.1960371732711792,
      "step": 2550
    },
    {
      "epoch": 1.34,
      "learning_rate": 1.4819063690713565e-06,
      "logits/chosen": 0.5778559446334839,
      "logits/rejected": 0.6045337915420532,
      "logps/chosen": -284.07061767578125,
      "logps/rejected": -270.4360656738281,
      "loss": 1938.5168,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12265495210886002,
      "rewards/margins": 0.09922391176223755,
      "rewards/rejected": -0.22187885642051697,
      "step": 2560
    },
    {
      "epoch": 1.35,
      "learning_rate": 1.4610846600005164e-06,
      "logits/chosen": 0.6385133862495422,
      "logits/rejected": 0.6164069175720215,
      "logps/chosen": -291.48590087890625,
      "logps/rejected": -241.2799072265625,
      "loss": 2040.5221,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.11285148561000824,
      "rewards/margins": 0.08863021433353424,
      "rewards/rejected": -0.20148172974586487,
      "step": 2570
    },
    {
      "epoch": 1.35,
      "learning_rate": 1.4403497000615885e-06,
      "logits/chosen": 0.6022018194198608,
      "logits/rejected": 0.6375949382781982,
      "logps/chosen": -250.8367156982422,
      "logps/rejected": -242.37881469726562,
      "loss": 2072.085,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.11340691894292831,
      "rewards/margins": 0.07300657033920288,
      "rewards/rejected": -0.1864134818315506,
      "step": 2580
    },
    {
      "epoch": 1.36,
      "learning_rate": 1.4197032206177618e-06,
      "logits/chosen": 0.6561594605445862,
      "logits/rejected": 0.7297431230545044,
      "logps/chosen": -249.74887084960938,
      "logps/rejected": -237.26779174804688,
      "loss": 2067.224,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.12197699397802353,
      "rewards/margins": 0.08562152087688446,
      "rewards/rejected": -0.2075985223054886,
      "step": 2590
    },
    {
      "epoch": 1.36,
      "learning_rate": 1.3991469456441273e-06,
      "logits/chosen": 0.6001744270324707,
      "logits/rejected": 0.6553865075111389,
      "logps/chosen": -252.2941436767578,
      "logps/rejected": -243.48025512695312,
      "loss": 1884.6086,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.11671899259090424,
      "rewards/margins": 0.10958864539861679,
      "rewards/rejected": -0.22630766034126282,
      "step": 2600
    },
    {
      "epoch": 1.36,
      "eval_logits/chosen": 0.545119047164917,
      "eval_logits/rejected": 0.601308286190033,
      "eval_logps/chosen": -268.77081298828125,
      "eval_logps/rejected": -254.6013946533203,
      "eval_loss": 2033.7977294921875,
      "eval_rewards/accuracies": 0.6909999847412109,
      "eval_rewards/chosen": -0.12153391540050507,
      "eval_rewards/margins": 0.08899100124835968,
      "eval_rewards/rejected": -0.21052493155002594,
      "eval_runtime": 416.4661,
      "eval_samples_per_second": 4.802,
      "eval_steps_per_second": 1.201,
      "step": 2600
    },
    {
      "epoch": 1.37,
      "learning_rate": 1.3786825915837299e-06,
      "logits/chosen": 0.6044927835464478,
      "logits/rejected": 0.608493447303772,
      "logps/chosen": -268.0179138183594,
      "logps/rejected": -251.26168823242188,
      "loss": 1763.491,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.08965936303138733,
      "rewards/margins": 0.12351739406585693,
      "rewards/rejected": -0.21317675709724426,
      "step": 2610
    },
    {
      "epoch": 1.37,
      "learning_rate": 1.3583118672042441e-06,
      "logits/chosen": 0.5879210233688354,
      "logits/rejected": 0.6254302263259888,
      "logps/chosen": -268.9471740722656,
      "logps/rejected": -246.9387664794922,
      "loss": 1806.3043,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.09920786321163177,
      "rewards/margins": 0.1189170852303505,
      "rewards/rejected": -0.21812494099140167,
      "step": 2620
    },
    {
      "epoch": 1.38,
      "learning_rate": 1.3380364734552935e-06,
      "logits/chosen": 0.6040158867835999,
      "logits/rejected": 0.6454821825027466,
      "logps/chosen": -239.55313110351562,
      "logps/rejected": -252.41641235351562,
      "loss": 1881.491,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11133377254009247,
      "rewards/margins": 0.10438641160726547,
      "rewards/rejected": -0.21572017669677734,
      "step": 2630
    },
    {
      "epoch": 1.38,
      "learning_rate": 1.3178581033264218e-06,
      "logits/chosen": 0.5422452688217163,
      "logits/rejected": 0.5576962232589722,
      "logps/chosen": -267.02020263671875,
      "logps/rejected": -233.59628295898438,
      "loss": 1922.1236,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.1054287701845169,
      "rewards/margins": 0.10195982456207275,
      "rewards/rejected": -0.20738859474658966,
      "step": 2640
    },
    {
      "epoch": 1.39,
      "learning_rate": 1.2977784417057262e-06,
      "logits/chosen": 0.5648713111877441,
      "logits/rejected": 0.5970919132232666,
      "logps/chosen": -266.724365234375,
      "logps/rejected": -252.4540557861328,
      "loss": 1788.8666,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.11531468480825424,
      "rewards/margins": 0.11205389350652695,
      "rewards/rejected": -0.22736859321594238,
      "step": 2650
    },
    {
      "epoch": 1.39,
      "learning_rate": 1.2777991652391757e-06,
      "logits/chosen": 0.5807424783706665,
      "logits/rejected": 0.64664626121521,
      "logps/chosen": -253.5209197998047,
      "logps/rejected": -251.5238800048828,
      "loss": 2035.5062,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.12570129334926605,
      "rewards/margins": 0.09053059667348862,
      "rewards/rejected": -0.21623189747333527,
      "step": 2660
    },
    {
      "epoch": 1.4,
      "learning_rate": 1.2579219421906049e-06,
      "logits/chosen": 0.612740159034729,
      "logits/rejected": 0.6295909285545349,
      "logps/chosen": -275.30938720703125,
      "logps/rejected": -246.85986328125,
      "loss": 1918.4975,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.11256787925958633,
      "rewards/margins": 0.1016065701842308,
      "rewards/rejected": -0.21417441964149475,
      "step": 2670
    },
    {
      "epoch": 1.4,
      "learning_rate": 1.2381484323024178e-06,
      "logits/chosen": 0.5338586568832397,
      "logits/rejected": 0.614523708820343,
      "logps/chosen": -248.32406616210938,
      "logps/rejected": -234.80862426757812,
      "loss": 2030.9229,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.12608769536018372,
      "rewards/margins": 0.09196772426366806,
      "rewards/rejected": -0.21805541217327118,
      "step": 2680
    },
    {
      "epoch": 1.41,
      "learning_rate": 1.2184802866569991e-06,
      "logits/chosen": 0.5740771889686584,
      "logits/rejected": 0.5626708269119263,
      "logps/chosen": -256.43524169921875,
      "logps/rejected": -254.8041534423828,
      "loss": 1881.1102,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.12152848392724991,
      "rewards/margins": 0.1093037948012352,
      "rewards/rejected": -0.2308322638273239,
      "step": 2690
    },
    {
      "epoch": 1.41,
      "learning_rate": 1.1989191475388518e-06,
      "logits/chosen": 0.5784533023834229,
      "logits/rejected": 0.5800845623016357,
      "logps/chosen": -261.47900390625,
      "logps/rejected": -261.27618408203125,
      "loss": 2034.9129,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12217319011688232,
      "rewards/margins": 0.08460931479930878,
      "rewards/rejected": -0.2067825049161911,
      "step": 2700
    },
    {
      "epoch": 1.41,
      "eval_logits/chosen": 0.5425635576248169,
      "eval_logits/rejected": 0.5986801385879517,
      "eval_logps/chosen": -268.96331787109375,
      "eval_logps/rejected": -254.94712829589844,
      "eval_loss": 2032.544677734375,
      "eval_rewards/accuracies": 0.6899999976158142,
      "eval_rewards/chosen": -0.12345867604017258,
      "eval_rewards/margins": 0.0905236080288887,
      "eval_rewards/rejected": -0.21398229897022247,
      "eval_runtime": 416.8138,
      "eval_samples_per_second": 4.798,
      "eval_steps_per_second": 1.2,
      "step": 2700
    },
    {
      "epoch": 1.42,
      "learning_rate": 1.1794666482974617e-06,
      "logits/chosen": 0.5704789161682129,
      "logits/rejected": 0.6782268285751343,
      "logps/chosen": -282.65875244140625,
      "logps/rejected": -257.2977600097656,
      "loss": 1989.7289,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.11434787511825562,
      "rewards/margins": 0.0926680713891983,
      "rewards/rejected": -0.20701594650745392,
      "step": 2710
    },
    {
      "epoch": 1.42,
      "learning_rate": 1.160124413210918e-06,
      "logits/chosen": 0.5337072014808655,
      "logits/rejected": 0.5367878675460815,
      "logps/chosen": -264.4056701660156,
      "logps/rejected": -245.18301391601562,
      "loss": 1913.7168,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.10953982174396515,
      "rewards/margins": 0.10163428634405136,
      "rewards/rejected": -0.2111741304397583,
      "step": 2720
    },
    {
      "epoch": 1.43,
      "learning_rate": 1.1408940573502838e-06,
      "logits/chosen": 0.5485426783561707,
      "logits/rejected": 0.6499109864234924,
      "logps/chosen": -264.18505859375,
      "logps/rejected": -238.2395477294922,
      "loss": 1907.4213,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.12582936882972717,
      "rewards/margins": 0.09968879073858261,
      "rewards/rejected": -0.22551818192005157,
      "step": 2730
    },
    {
      "epoch": 1.43,
      "learning_rate": 1.1217771864447396e-06,
      "logits/chosen": 0.5939881205558777,
      "logits/rejected": 0.6111994981765747,
      "logps/chosen": -261.0160827636719,
      "logps/rejected": -244.2744140625,
      "loss": 2013.2748,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.12591084837913513,
      "rewards/margins": 0.08752218633890152,
      "rewards/rejected": -0.21343302726745605,
      "step": 2740
    },
    {
      "epoch": 1.44,
      "learning_rate": 1.1027753967475046e-06,
      "logits/chosen": 0.5890164375305176,
      "logits/rejected": 0.6029259562492371,
      "logps/chosen": -259.98382568359375,
      "logps/rejected": -254.6918487548828,
      "loss": 1947.2754,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.1243789941072464,
      "rewards/margins": 0.09525910019874573,
      "rewards/rejected": -0.21963807940483093,
      "step": 2750
    },
    {
      "epoch": 1.44,
      "learning_rate": 1.08389027490255e-06,
      "logits/chosen": 0.5902668833732605,
      "logits/rejected": 0.6080381274223328,
      "logps/chosen": -248.3684844970703,
      "logps/rejected": -263.76776123046875,
      "loss": 2085.1875,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.13539178669452667,
      "rewards/margins": 0.07983563095331192,
      "rewards/rejected": -0.215227410197258,
      "step": 2760
    },
    {
      "epoch": 1.45,
      "learning_rate": 1.0651233978121145e-06,
      "logits/chosen": 0.5521366596221924,
      "logits/rejected": 0.5906900763511658,
      "logps/chosen": -300.05230712890625,
      "logps/rejected": -272.1240234375,
      "loss": 1883.4229,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.11396439373493195,
      "rewards/margins": 0.09765832126140594,
      "rewards/rejected": -0.2116227149963379,
      "step": 2770
    },
    {
      "epoch": 1.46,
      "learning_rate": 1.046476332505036e-06,
      "logits/chosen": 0.6347781419754028,
      "logits/rejected": 0.6657929420471191,
      "logps/chosen": -250.8879852294922,
      "logps/rejected": -223.33255004882812,
      "loss": 1819.1854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11131460964679718,
      "rewards/margins": 0.11518070846796036,
      "rewards/rejected": -0.22649531066417694,
      "step": 2780
    },
    {
      "epoch": 1.46,
      "learning_rate": 1.0279506360059005e-06,
      "logits/chosen": 0.5551185011863708,
      "logits/rejected": 0.5792326331138611,
      "logps/chosen": -262.4249572753906,
      "logps/rejected": -267.63763427734375,
      "loss": 2209.3523,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.12959793210029602,
      "rewards/margins": 0.06600113213062286,
      "rewards/rejected": -0.19559906423091888,
      "step": 2790
    },
    {
      "epoch": 1.47,
      "learning_rate": 1.0095478552050348e-06,
      "logits/chosen": 0.6019959449768066,
      "logits/rejected": 0.6074908971786499,
      "logps/chosen": -273.5536193847656,
      "logps/rejected": -267.5385437011719,
      "loss": 2068.2822,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.13250732421875,
      "rewards/margins": 0.09504042565822601,
      "rewards/rejected": -0.2275477647781372,
      "step": 2800
    },
    {
      "epoch": 1.47,
      "eval_logits/chosen": 0.5382584929466248,
      "eval_logits/rejected": 0.5942660570144653,
      "eval_logps/chosen": -269.1269836425781,
      "eval_logps/rejected": -255.16705322265625,
      "eval_loss": 2030.8697509765625,
      "eval_rewards/accuracies": 0.6899999976158142,
      "eval_rewards/chosen": -0.1250954419374466,
      "eval_rewards/margins": 0.09108588099479675,
      "eval_rewards/rejected": -0.21618132293224335,
      "eval_runtime": 416.6389,
      "eval_samples_per_second": 4.8,
      "eval_steps_per_second": 1.2,
      "step": 2800
    },
    {
      "epoch": 1.47,
      "learning_rate": 9.912695267293383e-07,
      "logits/chosen": 0.5214653015136719,
      "logits/rejected": 0.5876752734184265,
      "logps/chosen": -265.47882080078125,
      "logps/rejected": -239.1663360595703,
      "loss": 1950.4564,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10924456268548965,
      "rewards/margins": 0.09817437827587128,
      "rewards/rejected": -0.20741891860961914,
      "step": 2810
    },
    {
      "epoch": 1.48,
      "learning_rate": 9.731171768139808e-07,
      "logits/chosen": 0.6136573553085327,
      "logits/rejected": 0.6188865900039673,
      "logps/chosen": -284.9826965332031,
      "logps/rejected": -261.3671875,
      "loss": 2210.9309,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.12047699838876724,
      "rewards/margins": 0.06873573362827301,
      "rewards/rejected": -0.18921272456645966,
      "step": 2820
    },
    {
      "epoch": 1.48,
      "learning_rate": 9.550923211749557e-07,
      "logits/chosen": 0.5326896905899048,
      "logits/rejected": 0.5845073461532593,
      "logps/chosen": -260.52069091796875,
      "logps/rejected": -268.2504577636719,
      "loss": 2028.4584,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.11890892684459686,
      "rewards/margins": 0.08525559306144714,
      "rewards/rejected": -0.2041645348072052,
      "step": 2830
    },
    {
      "epoch": 1.49,
      "learning_rate": 9.371964648825221e-07,
      "logits/chosen": 0.6162235736846924,
      "logits/rejected": 0.5728213787078857,
      "logps/chosen": -272.336181640625,
      "logps/rejected": -252.33346557617188,
      "loss": 1941.8066,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.10710117965936661,
      "rewards/margins": 0.10685823112726212,
      "rewards/rejected": -0.21395941078662872,
      "step": 2840
    },
    {
      "epoch": 1.49,
      "learning_rate": 9.194311022355279e-07,
      "logits/chosen": 0.5015624761581421,
      "logits/rejected": 0.5448901057243347,
      "logps/chosen": -276.95538330078125,
      "logps/rejected": -250.8726348876953,
      "loss": 1832.3256,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.10919035971164703,
      "rewards/margins": 0.11640901863574982,
      "rewards/rejected": -0.22559937834739685,
      "step": 2850
    },
    {
      "epoch": 1.5,
      "learning_rate": 9.017977166366445e-07,
      "logits/chosen": 0.5708821415901184,
      "logits/rejected": 0.558485209941864,
      "logps/chosen": -258.96807861328125,
      "logps/rejected": -263.4228210449219,
      "loss": 1947.8791,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11477123200893402,
      "rewards/margins": 0.09229175001382828,
      "rewards/rejected": -0.2070629894733429,
      "step": 2860
    },
    {
      "epoch": 1.5,
      "learning_rate": 8.842977804684938e-07,
      "logits/chosen": 0.5845485925674438,
      "logits/rejected": 0.6778086423873901,
      "logps/chosen": -245.46102905273438,
      "logps/rejected": -233.6326141357422,
      "loss": 2066.3828,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.12042129039764404,
      "rewards/margins": 0.08121231943368912,
      "rewards/rejected": -0.20163361728191376,
      "step": 2870
    },
    {
      "epoch": 1.51,
      "learning_rate": 8.669327549707096e-07,
      "logits/chosen": 0.5501264929771423,
      "logits/rejected": 0.6287878751754761,
      "logps/chosen": -281.37371826171875,
      "logps/rejected": -252.8254852294922,
      "loss": 1868.0959,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1148761659860611,
      "rewards/margins": 0.1036820039153099,
      "rewards/rejected": -0.2185581624507904,
      "step": 2880
    },
    {
      "epoch": 1.51,
      "learning_rate": 8.497040901179232e-07,
      "logits/chosen": 0.5025564432144165,
      "logits/rejected": 0.5421415567398071,
      "logps/chosen": -276.861572265625,
      "logps/rejected": -267.47723388671875,
      "loss": 1753.2416,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.10830195993185043,
      "rewards/margins": 0.12366169691085815,
      "rewards/rejected": -0.2319636344909668,
      "step": 2890
    },
    {
      "epoch": 1.52,
      "learning_rate": 8.326132244986932e-07,
      "logits/chosen": 0.6039875745773315,
      "logits/rejected": 0.6563787460327148,
      "logps/chosen": -282.04266357421875,
      "logps/rejected": -257.56622314453125,
      "loss": 1977.4029,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.12028930336236954,
      "rewards/margins": 0.09056351333856583,
      "rewards/rejected": -0.21085281670093536,
      "step": 2900
    },
    {
      "epoch": 1.52,
      "eval_logits/chosen": 0.5381389260292053,
      "eval_logits/rejected": 0.5940784811973572,
      "eval_logps/chosen": -269.125244140625,
      "eval_logps/rejected": -255.16897583007812,
      "eval_loss": 2030.603271484375,
      "eval_rewards/accuracies": 0.6894999742507935,
      "eval_rewards/chosen": -0.12507818639278412,
      "eval_rewards/margins": 0.09112255275249481,
      "eval_rewards/rejected": -0.21620073914527893,
      "eval_runtime": 416.5425,
      "eval_samples_per_second": 4.801,
      "eval_steps_per_second": 1.2,
      "step": 2900
    },
    {
      "epoch": 1.52,
      "learning_rate": 8.156615851953798e-07,
      "logits/chosen": 0.559486448764801,
      "logits/rejected": 0.5794366598129272,
      "logps/chosen": -256.5633239746094,
      "logps/rejected": -259.57696533203125,
      "loss": 1948.7941,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.10746540874242783,
      "rewards/margins": 0.10354423522949219,
      "rewards/rejected": -0.2110096514225006,
      "step": 2910
    },
    {
      "epoch": 1.53,
      "learning_rate": 7.988505876649863e-07,
      "logits/chosen": 0.6247807741165161,
      "logits/rejected": 0.6010321974754333,
      "logps/chosen": -271.8721923828125,
      "logps/rejected": -256.86480712890625,
      "loss": 2099.2482,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12578730285167694,
      "rewards/margins": 0.084382563829422,
      "rewards/rejected": -0.21016988158226013,
      "step": 2920
    },
    {
      "epoch": 1.53,
      "learning_rate": 7.821816356209677e-07,
      "logits/chosen": 0.5775936841964722,
      "logits/rejected": 0.6070097088813782,
      "logps/chosen": -272.50653076171875,
      "logps/rejected": -251.46243286132812,
      "loss": 2020.2645,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10367625951766968,
      "rewards/margins": 0.08927679061889648,
      "rewards/rejected": -0.19295303523540497,
      "step": 2930
    },
    {
      "epoch": 1.54,
      "learning_rate": 7.656561209160248e-07,
      "logits/chosen": 0.521769642829895,
      "logits/rejected": 0.5275167226791382,
      "logps/chosen": -289.0915222167969,
      "logps/rejected": -263.7314453125,
      "loss": 1947.6189,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.12388887256383896,
      "rewards/margins": 0.1033380776643753,
      "rewards/rejected": -0.22722692787647247,
      "step": 2940
    },
    {
      "epoch": 1.54,
      "learning_rate": 7.492754234258794e-07,
      "logits/chosen": 0.5926128625869751,
      "logits/rejected": 0.6193209886550903,
      "logps/chosen": -241.3407440185547,
      "logps/rejected": -225.17724609375,
      "loss": 1876.9375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.10547138750553131,
      "rewards/margins": 0.10478665679693222,
      "rewards/rejected": -0.21025805175304413,
      "step": 2950
    },
    {
      "epoch": 1.55,
      "learning_rate": 7.330409109340563e-07,
      "logits/chosen": 0.5721119046211243,
      "logits/rejected": 0.5718821287155151,
      "logps/chosen": -267.39471435546875,
      "logps/rejected": -244.5928192138672,
      "loss": 2126.3432,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.1287916600704193,
      "rewards/margins": 0.07899868488311768,
      "rewards/rejected": -0.2077903300523758,
      "step": 2960
    },
    {
      "epoch": 1.55,
      "learning_rate": 7.169539390176769e-07,
      "logits/chosen": 0.5741583704948425,
      "logits/rejected": 0.5660156011581421,
      "logps/chosen": -219.59640502929688,
      "logps/rejected": -233.7797088623047,
      "loss": 1845.3854,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.12960752844810486,
      "rewards/margins": 0.10263533890247345,
      "rewards/rejected": -0.23224285244941711,
      "step": 2970
    },
    {
      "epoch": 1.56,
      "learning_rate": 7.010158509342682e-07,
      "logits/chosen": 0.5922077298164368,
      "logits/rejected": 0.6388793587684631,
      "logps/chosen": -258.38946533203125,
      "logps/rejected": -236.8799285888672,
      "loss": 1677.0656,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.11209128051996231,
      "rewards/margins": 0.13626167178153992,
      "rewards/rejected": -0.24835292994976044,
      "step": 2980
    },
    {
      "epoch": 1.57,
      "learning_rate": 6.852279775095976e-07,
      "logits/chosen": 0.6180992722511292,
      "logits/rejected": 0.6189014911651611,
      "logps/chosen": -272.6584167480469,
      "logps/rejected": -247.75033569335938,
      "loss": 1925.8682,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.11567474901676178,
      "rewards/margins": 0.09763548523187637,
      "rewards/rejected": -0.21331021189689636,
      "step": 2990
    },
    {
      "epoch": 1.57,
      "learning_rate": 6.695916370265529e-07,
      "logits/chosen": 0.6014515161514282,
      "logits/rejected": 0.5875986814498901,
      "logps/chosen": -265.0668029785156,
      "logps/rejected": -241.8825225830078,
      "loss": 2110.2887,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.1249975711107254,
      "rewards/margins": 0.07782919704914093,
      "rewards/rejected": -0.20282676815986633,
      "step": 3000
    },
    {
      "epoch": 1.57,
      "eval_logits/chosen": 0.5348395109176636,
      "eval_logits/rejected": 0.5908406972885132,
      "eval_logps/chosen": -269.2049865722656,
      "eval_logps/rejected": -255.2820587158203,
      "eval_loss": 2030.5706787109375,
      "eval_rewards/accuracies": 0.690500020980835,
      "eval_rewards/chosen": -0.12587547302246094,
      "eval_rewards/margins": 0.09145611524581909,
      "eval_rewards/rejected": -0.21733158826828003,
      "eval_runtime": 416.6652,
      "eval_samples_per_second": 4.8,
      "eval_steps_per_second": 1.2,
      "step": 3000
    },
    {
      "epoch": 1.58,
      "learning_rate": 6.541081351150638e-07,
      "logits/chosen": 0.5409640669822693,
      "logits/rejected": 0.5331202149391174,
      "logps/chosen": -279.83941650390625,
      "logps/rejected": -291.9646301269531,
      "loss": 2035.416,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.10843896865844727,
      "rewards/margins": 0.09788022935390472,
      "rewards/rejected": -0.2063191831111908,
      "step": 3010
    },
    {
      "epoch": 1.58,
      "learning_rate": 6.387787646430854e-07,
      "logits/chosen": 0.5321037769317627,
      "logits/rejected": 0.5582699775695801,
      "logps/chosen": -267.9813232421875,
      "logps/rejected": -264.12567138671875,
      "loss": 2006.4391,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.10174393653869629,
      "rewards/margins": 0.08532971143722534,
      "rewards/rejected": -0.18707364797592163,
      "step": 3020
    },
    {
      "epoch": 1.59,
      "learning_rate": 6.2360480560864e-07,
      "logits/chosen": 0.5698617696762085,
      "logits/rejected": 0.5839768648147583,
      "logps/chosen": -251.5703125,
      "logps/rejected": -235.4780731201172,
      "loss": 1821.7498,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.10450971126556396,
      "rewards/margins": 0.10978861898183823,
      "rewards/rejected": -0.2142982929944992,
      "step": 3030
    },
    {
      "epoch": 1.59,
      "learning_rate": 6.085875250329401e-07,
      "logits/chosen": 0.5382856726646423,
      "logits/rejected": 0.6018794178962708,
      "logps/chosen": -304.603271484375,
      "logps/rejected": -263.90570068359375,
      "loss": 1820.3352,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10189330577850342,
      "rewards/margins": 0.12546256184577942,
      "rewards/rejected": -0.22735583782196045,
      "step": 3040
    },
    {
      "epoch": 1.6,
      "learning_rate": 5.937281768545919e-07,
      "logits/chosen": 0.600039005279541,
      "logits/rejected": 0.5895189046859741,
      "logps/chosen": -288.302734375,
      "logps/rejected": -266.3108215332031,
      "loss": 2142.2947,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1374974250793457,
      "rewards/margins": 0.08518020063638687,
      "rewards/rejected": -0.22267761826515198,
      "step": 3050
    },
    {
      "epoch": 1.6,
      "learning_rate": 5.79028001824894e-07,
      "logits/chosen": 0.577072024345398,
      "logits/rejected": 0.5782276391983032,
      "logps/chosen": -258.71734619140625,
      "logps/rejected": -252.88906860351562,
      "loss": 2083.8391,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.13044390082359314,
      "rewards/margins": 0.09016549587249756,
      "rewards/rejected": -0.22060942649841309,
      "step": 3060
    },
    {
      "epoch": 1.61,
      "learning_rate": 5.644882274042285e-07,
      "logits/chosen": 0.5784581303596497,
      "logits/rejected": 0.5805580019950867,
      "logps/chosen": -286.2455139160156,
      "logps/rejected": -253.4708251953125,
      "loss": 1979.6775,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11061519384384155,
      "rewards/margins": 0.10562906414270401,
      "rewards/rejected": -0.21624425053596497,
      "step": 3070
    },
    {
      "epoch": 1.61,
      "learning_rate": 5.501100676595761e-07,
      "logits/chosen": 0.5630078911781311,
      "logits/rejected": 0.5682691335678101,
      "logps/chosen": -267.6597900390625,
      "logps/rejected": -250.3827667236328,
      "loss": 1964.8426,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1262790709733963,
      "rewards/margins": 0.09482350945472717,
      "rewards/rejected": -0.22110256552696228,
      "step": 3080
    },
    {
      "epoch": 1.62,
      "learning_rate": 5.358947231631375e-07,
      "logits/chosen": 0.534908652305603,
      "logits/rejected": 0.5746644139289856,
      "logps/chosen": -283.8402404785156,
      "logps/rejected": -272.68670654296875,
      "loss": 1792.1418,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0998634472489357,
      "rewards/margins": 0.1285194605588913,
      "rewards/rejected": -0.2283829152584076,
      "step": 3090
    },
    {
      "epoch": 1.62,
      "learning_rate": 5.218433808920884e-07,
      "logits/chosen": 0.5141295194625854,
      "logits/rejected": 0.5280352234840393,
      "logps/chosen": -262.8772888183594,
      "logps/rejected": -246.84671020507812,
      "loss": 2068.2863,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.10809364169836044,
      "rewards/margins": 0.08412571996450424,
      "rewards/rejected": -0.19221936166286469,
      "step": 3100
    },
    {
      "epoch": 1.62,
      "eval_logits/chosen": 0.5356869697570801,
      "eval_logits/rejected": 0.5913118720054626,
      "eval_logps/chosen": -269.03900146484375,
      "eval_logps/rejected": -255.10865783691406,
      "eval_loss": 2029.4173583984375,
      "eval_rewards/accuracies": 0.6934999823570251,
      "eval_rewards/chosen": -0.12421557307243347,
      "eval_rewards/margins": 0.09138190746307373,
      "eval_rewards/rejected": -0.2155974805355072,
      "eval_runtime": 416.645,
      "eval_samples_per_second": 4.8,
      "eval_steps_per_second": 1.2,
      "step": 3100
    },
    {
      "epoch": 1.63,
      "learning_rate": 5.07957214129464e-07,
      "logits/chosen": 0.6343733072280884,
      "logits/rejected": 0.6377061605453491,
      "logps/chosen": -230.1392059326172,
      "logps/rejected": -217.2322540283203,
      "loss": 2110.5152,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.14483976364135742,
      "rewards/margins": 0.082811638712883,
      "rewards/rejected": -0.22765140235424042,
      "step": 3110
    },
    {
      "epoch": 1.63,
      "learning_rate": 4.942373823661928e-07,
      "logits/chosen": 0.5317670702934265,
      "logits/rejected": 0.5477628707885742,
      "logps/chosen": -253.5054931640625,
      "logps/rejected": -295.16021728515625,
      "loss": 2379.184,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.13306215405464172,
      "rewards/margins": 0.05652584508061409,
      "rewards/rejected": -0.18958799540996552,
      "step": 3120
    },
    {
      "epoch": 1.64,
      "learning_rate": 4.806850312042782e-07,
      "logits/chosen": 0.6451593637466431,
      "logits/rejected": 0.5899637937545776,
      "logps/chosen": -289.49151611328125,
      "logps/rejected": -257.98443603515625,
      "loss": 1992.5604,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.12486691772937775,
      "rewards/margins": 0.09148009121417999,
      "rewards/rejected": -0.21634697914123535,
      "step": 3130
    },
    {
      "epoch": 1.64,
      "learning_rate": 4.6730129226114363e-07,
      "logits/chosen": 0.5886529684066772,
      "logits/rejected": 0.5409609079360962,
      "logps/chosen": -258.1368103027344,
      "logps/rejected": -249.2908935546875,
      "loss": 1995.9428,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11228666454553604,
      "rewards/margins": 0.08399216085672379,
      "rewards/rejected": -0.19627881050109863,
      "step": 3140
    },
    {
      "epoch": 1.65,
      "learning_rate": 4.540872830751386e-07,
      "logits/chosen": 0.5374349355697632,
      "logits/rejected": 0.5601732134819031,
      "logps/chosen": -266.9260559082031,
      "logps/rejected": -266.8581237792969,
      "loss": 2206.0141,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12373526394367218,
      "rewards/margins": 0.060657333582639694,
      "rewards/rejected": -0.18439260125160217,
      "step": 3150
    },
    {
      "epoch": 1.65,
      "learning_rate": 4.4104410701222703e-07,
      "logits/chosen": 0.5198964476585388,
      "logits/rejected": 0.5673514008522034,
      "logps/chosen": -250.6959228515625,
      "logps/rejected": -233.34774780273438,
      "loss": 1759.9059,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.11838851869106293,
      "rewards/margins": 0.12481342256069183,
      "rewards/rejected": -0.24320194125175476,
      "step": 3160
    },
    {
      "epoch": 1.66,
      "learning_rate": 4.281728531738563e-07,
      "logits/chosen": 0.597510814666748,
      "logits/rejected": 0.6312834620475769,
      "logps/chosen": -268.5950012207031,
      "logps/rejected": -250.58059692382812,
      "loss": 1960.2506,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11448697745800018,
      "rewards/margins": 0.09015806764364243,
      "rewards/rejected": -0.204645037651062,
      "step": 3170
    },
    {
      "epoch": 1.66,
      "learning_rate": 4.154745963060197e-07,
      "logits/chosen": 0.507027268409729,
      "logits/rejected": 0.5732488632202148,
      "logps/chosen": -280.98382568359375,
      "logps/rejected": -286.677734375,
      "loss": 1965.5633,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.12583956122398376,
      "rewards/margins": 0.10458560287952423,
      "rewards/rejected": -0.230425164103508,
      "step": 3180
    },
    {
      "epoch": 1.67,
      "learning_rate": 4.029503967095097e-07,
      "logits/chosen": 0.4729984402656555,
      "logits/rejected": 0.5827825665473938,
      "logps/chosen": -268.3514709472656,
      "logps/rejected": -247.7762451171875,
      "loss": 1878.176,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.1029181256890297,
      "rewards/margins": 0.09399056434631348,
      "rewards/rejected": -0.19690869748592377,
      "step": 3190
    },
    {
      "epoch": 1.67,
      "learning_rate": 3.9060130015138863e-07,
      "logits/chosen": 0.5795052647590637,
      "logits/rejected": 0.6329609155654907,
      "logps/chosen": -262.9614562988281,
      "logps/rejected": -240.7771453857422,
      "loss": 1977.8852,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1271854192018509,
      "rewards/margins": 0.08980287611484528,
      "rewards/rejected": -0.21698825061321259,
      "step": 3200
    },
    {
      "epoch": 1.67,
      "eval_logits/chosen": 0.5363709926605225,
      "eval_logits/rejected": 0.5920352935791016,
      "eval_logps/chosen": -269.10711669921875,
      "eval_logps/rejected": -255.2016143798828,
      "eval_loss": 2026.12890625,
      "eval_rewards/accuracies": 0.6959999799728394,
      "eval_rewards/chosen": -0.1248970478773117,
      "eval_rewards/margins": 0.09163003414869308,
      "eval_rewards/rejected": -0.21652711927890778,
      "eval_runtime": 416.4729,
      "eval_samples_per_second": 4.802,
      "eval_steps_per_second": 1.201,
      "step": 3200
    },
    {
      "epoch": 1.68,
      "learning_rate": 3.784283377776651e-07,
      "logits/chosen": 0.6236351728439331,
      "logits/rejected": 0.630204975605011,
      "logps/chosen": -267.3162536621094,
      "logps/rejected": -241.71484375,
      "loss": 2151.366,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.13855421543121338,
      "rewards/margins": 0.07831588387489319,
      "rewards/rejected": -0.21687009930610657,
      "step": 3210
    },
    {
      "epoch": 1.69,
      "learning_rate": 3.664325260271953e-07,
      "logits/chosen": 0.586463451385498,
      "logits/rejected": 0.6219819784164429,
      "logps/chosen": -240.7449493408203,
      "logps/rejected": -260.012451171875,
      "loss": 2145.4057,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.12818796932697296,
      "rewards/margins": 0.07563059777021408,
      "rewards/rejected": -0.20381855964660645,
      "step": 3220
    },
    {
      "epoch": 1.69,
      "learning_rate": 3.5461486654680746e-07,
      "logits/chosen": 0.5574949979782104,
      "logits/rejected": 0.6360602378845215,
      "logps/chosen": -262.3418884277344,
      "logps/rejected": -256.82366943359375,
      "loss": 2092.1422,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.10938684642314911,
      "rewards/margins": 0.08223484456539154,
      "rewards/rejected": -0.19162169098854065,
      "step": 3230
    },
    {
      "epoch": 1.7,
      "learning_rate": 3.429763461076677e-07,
      "logits/chosen": 0.5418500304222107,
      "logits/rejected": 0.5625206828117371,
      "logps/chosen": -271.43792724609375,
      "logps/rejected": -255.0489501953125,
      "loss": 2035.8221,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13326099514961243,
      "rewards/margins": 0.08890596777200699,
      "rewards/rejected": -0.2221669703722,
      "step": 3240
    },
    {
      "epoch": 1.7,
      "learning_rate": 3.315179365228824e-07,
      "logits/chosen": 0.5612285733222961,
      "logits/rejected": 0.5996168851852417,
      "logps/chosen": -284.46612548828125,
      "logps/rejected": -261.4803771972656,
      "loss": 1981.793,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.12592165172100067,
      "rewards/margins": 0.0976746454834938,
      "rewards/rejected": -0.22359630465507507,
      "step": 3250
    },
    {
      "epoch": 1.71,
      "learning_rate": 3.202405945663556e-07,
      "logits/chosen": 0.5869094729423523,
      "logits/rejected": 0.5811904668807983,
      "logps/chosen": -273.99249267578125,
      "logps/rejected": -274.64410400390625,
      "loss": 2093.5598,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11422860622406006,
      "rewards/margins": 0.09214137494564056,
      "rewards/rejected": -0.20636996626853943,
      "step": 3260
    },
    {
      "epoch": 1.71,
      "learning_rate": 3.09145261892895e-07,
      "logits/chosen": 0.5232716798782349,
      "logits/rejected": 0.6522939801216125,
      "logps/chosen": -266.1856384277344,
      "logps/rejected": -255.4128875732422,
      "loss": 1884.3191,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.12048964202404022,
      "rewards/margins": 0.1096540093421936,
      "rewards/rejected": -0.23014366626739502,
      "step": 3270
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.982328649595856e-07,
      "logits/chosen": 0.5434025526046753,
      "logits/rejected": 0.5516559481620789,
      "logps/chosen": -261.6842346191406,
      "logps/rejected": -268.0997619628906,
      "loss": 2020.6832,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.12252092361450195,
      "rewards/margins": 0.08777900040149689,
      "rewards/rejected": -0.21029992401599884,
      "step": 3280
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.8750431494843076e-07,
      "logits/chosen": 0.5793955326080322,
      "logits/rejected": 0.5846759676933289,
      "logps/chosen": -256.3296813964844,
      "logps/rejected": -258.81512451171875,
      "loss": 2103.4049,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.13012662529945374,
      "rewards/margins": 0.07379056513309479,
      "rewards/rejected": -0.20391719043254852,
      "step": 3290
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.7696050769026954e-07,
      "logits/chosen": 0.5790996551513672,
      "logits/rejected": 0.5938167572021484,
      "logps/chosen": -229.91567993164062,
      "logps/rejected": -236.54916381835938,
      "loss": 2123.3787,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.12688665091991425,
      "rewards/margins": 0.07247930765151978,
      "rewards/rejected": -0.19936595857143402,
      "step": 3300
    },
    {
      "epoch": 1.73,
      "eval_logits/chosen": 0.5370410680770874,
      "eval_logits/rejected": 0.5926198363304138,
      "eval_logps/chosen": -269.0932922363281,
      "eval_logps/rejected": -255.16659545898438,
      "eval_loss": 2027.355224609375,
      "eval_rewards/accuracies": 0.6930000185966492,
      "eval_rewards/chosen": -0.12475859373807907,
      "eval_rewards/margins": 0.09141821414232254,
      "eval_rewards/rejected": -0.2161768227815628,
      "eval_runtime": 416.6319,
      "eval_samples_per_second": 4.8,
      "eval_steps_per_second": 1.2,
      "step": 3300
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.666023235899734e-07,
      "logits/chosen": 0.5439051389694214,
      "logits/rejected": 0.638985276222229,
      "logps/chosen": -249.70217895507812,
      "logps/rejected": -246.07040405273438,
      "loss": 1936.2746,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.13094648718833923,
      "rewards/margins": 0.10491780191659927,
      "rewards/rejected": -0.23586425185203552,
      "step": 3310
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.564306275529341e-07,
      "logits/chosen": 0.5696260929107666,
      "logits/rejected": 0.6271142959594727,
      "logps/chosen": -288.08721923828125,
      "logps/rejected": -263.6356201171875,
      "loss": 1974.6082,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.11936695873737335,
      "rewards/margins": 0.10857198387384415,
      "rewards/rejected": -0.2279389351606369,
      "step": 3320
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.4644626891284243e-07,
      "logits/chosen": 0.5715283155441284,
      "logits/rejected": 0.6530539393424988,
      "logps/chosen": -245.0167236328125,
      "logps/rejected": -238.81460571289062,
      "loss": 2066.4543,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.12561528384685516,
      "rewards/margins": 0.08124671876430511,
      "rewards/rejected": -0.20686200261116028,
      "step": 3330
    },
    {
      "epoch": 1.75,
      "learning_rate": 2.3665008136077332e-07,
      "logits/chosen": 0.5698996186256409,
      "logits/rejected": 0.6029760837554932,
      "logps/chosen": -264.3577575683594,
      "logps/rejected": -271.068115234375,
      "loss": 2090.4994,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1360047161579132,
      "rewards/margins": 0.08008682727813721,
      "rewards/rejected": -0.2160915583372116,
      "step": 3340
    },
    {
      "epoch": 1.75,
      "learning_rate": 2.2704288287556718e-07,
      "logits/chosen": 0.5687640309333801,
      "logits/rejected": 0.5940347909927368,
      "logps/chosen": -257.6128845214844,
      "logps/rejected": -248.91015625,
      "loss": 2136.7539,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12395147979259491,
      "rewards/margins": 0.0794539600610733,
      "rewards/rejected": -0.2034054547548294,
      "step": 3350
    },
    {
      "epoch": 1.76,
      "learning_rate": 2.1762547565553293e-07,
      "logits/chosen": 0.5388206839561462,
      "logits/rejected": 0.5561047792434692,
      "logps/chosen": -261.9334716796875,
      "logps/rejected": -260.14556884765625,
      "loss": 1989.5404,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1216350793838501,
      "rewards/margins": 0.09191958606243134,
      "rewards/rejected": -0.21355466544628143,
      "step": 3360
    },
    {
      "epoch": 1.76,
      "learning_rate": 2.083986460514631e-07,
      "logits/chosen": 0.5701113343238831,
      "logits/rejected": 0.6196510195732117,
      "logps/chosen": -251.4109344482422,
      "logps/rejected": -252.1616668701172,
      "loss": 1820.4641,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.12642905116081238,
      "rewards/margins": 0.10934920608997345,
      "rewards/rejected": -0.23577824234962463,
      "step": 3370
    },
    {
      "epoch": 1.77,
      "learning_rate": 1.993631645009747e-07,
      "logits/chosen": 0.5514119863510132,
      "logits/rejected": 0.5474542379379272,
      "logps/chosen": -256.45831298828125,
      "logps/rejected": -227.55514526367188,
      "loss": 1845.2707,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.10441195964813232,
      "rewards/margins": 0.1090712919831276,
      "rewards/rejected": -0.21348324418067932,
      "step": 3380
    },
    {
      "epoch": 1.77,
      "learning_rate": 1.9051978546417715e-07,
      "logits/chosen": 0.5202070474624634,
      "logits/rejected": 0.5690991282463074,
      "logps/chosen": -260.46600341796875,
      "logps/rejected": -261.4082946777344,
      "loss": 1912.7502,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.10940267145633698,
      "rewards/margins": 0.10180971771478653,
      "rewards/rejected": -0.2112123966217041,
      "step": 3390
    },
    {
      "epoch": 1.78,
      "learning_rate": 1.818692473606748e-07,
      "logits/chosen": 0.5479332208633423,
      "logits/rejected": 0.5715588331222534,
      "logps/chosen": -258.5849609375,
      "logps/rejected": -264.29388427734375,
      "loss": 1945.4934,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.12341777980327606,
      "rewards/margins": 0.09188680350780487,
      "rewards/rejected": -0.21530456840991974,
      "step": 3400
    },
    {
      "epoch": 1.78,
      "eval_logits/chosen": 0.5352820754051208,
      "eval_logits/rejected": 0.5908908247947693,
      "eval_logps/chosen": -269.1009826660156,
      "eval_logps/rejected": -255.1898651123047,
      "eval_loss": 2025.7803955078125,
      "eval_rewards/accuracies": 0.6934999823570251,
      "eval_rewards/chosen": -0.12483509629964828,
      "eval_rewards/margins": 0.09157437831163406,
      "eval_rewards/rejected": -0.21640948951244354,
      "eval_runtime": 416.572,
      "eval_samples_per_second": 4.801,
      "eval_steps_per_second": 1.2,
      "step": 3400
    },
    {
      "epoch": 1.78,
      "learning_rate": 1.7341227250790989e-07,
      "logits/chosen": 0.5836583375930786,
      "logits/rejected": 0.632857084274292,
      "logps/chosen": -245.8205108642578,
      "logps/rejected": -252.48471069335938,
      "loss": 1828.1664,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.10635235160589218,
      "rewards/margins": 0.11655166000127792,
      "rewards/rejected": -0.2229039967060089,
      "step": 3410
    },
    {
      "epoch": 1.79,
      "learning_rate": 1.6514956706084885e-07,
      "logits/chosen": 0.6221760511398315,
      "logits/rejected": 0.5567342042922974,
      "logps/chosen": -266.02239990234375,
      "logps/rejected": -246.35385131835938,
      "loss": 1826.1057,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09994658827781677,
      "rewards/margins": 0.1130019798874855,
      "rewards/rejected": -0.21294856071472168,
      "step": 3420
    },
    {
      "epoch": 1.8,
      "learning_rate": 1.5708182095301867e-07,
      "logits/chosen": 0.6005284190177917,
      "logits/rejected": 0.6083909869194031,
      "logps/chosen": -280.53741455078125,
      "logps/rejected": -261.88201904296875,
      "loss": 1851.2512,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.11887475103139877,
      "rewards/margins": 0.10133900493383408,
      "rewards/rejected": -0.22021374106407166,
      "step": 3430
    },
    {
      "epoch": 1.8,
      "learning_rate": 1.4920970783889737e-07,
      "logits/chosen": 0.5680890083312988,
      "logits/rejected": 0.5507141351699829,
      "logps/chosen": -271.96990966796875,
      "logps/rejected": -241.1654815673828,
      "loss": 2041.0072,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.1112411841750145,
      "rewards/margins": 0.08822907507419586,
      "rewards/rejected": -0.19947026669979095,
      "step": 3440
    },
    {
      "epoch": 1.81,
      "learning_rate": 1.4153388503766492e-07,
      "logits/chosen": 0.5438860654830933,
      "logits/rejected": 0.5644111633300781,
      "logps/chosen": -279.3092346191406,
      "logps/rejected": -239.37167358398438,
      "loss": 1966.0102,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.1369594931602478,
      "rewards/margins": 0.09016064554452896,
      "rewards/rejected": -0.22712013125419617,
      "step": 3450
    },
    {
      "epoch": 1.81,
      "learning_rate": 1.340549934783164e-07,
      "logits/chosen": 0.6110261082649231,
      "logits/rejected": 0.6002285480499268,
      "logps/chosen": -255.5424346923828,
      "logps/rejected": -258.6153259277344,
      "loss": 1778.4668,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.10746339708566666,
      "rewards/margins": 0.12720224261283875,
      "rewards/rejected": -0.23466560244560242,
      "step": 3460
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.2677365764614452e-07,
      "logits/chosen": 0.6116484999656677,
      "logits/rejected": 0.6142521500587463,
      "logps/chosen": -251.9376983642578,
      "logps/rejected": -247.8961639404297,
      "loss": 1943.7922,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.12416845560073853,
      "rewards/margins": 0.09600269794464111,
      "rewards/rejected": -0.22017112374305725,
      "step": 3470
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.196904855305961e-07,
      "logits/chosen": 0.5488280057907104,
      "logits/rejected": 0.6329927444458008,
      "logps/chosen": -261.445068359375,
      "logps/rejected": -255.37142944335938,
      "loss": 2087.1572,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11578680574893951,
      "rewards/margins": 0.08292602747678757,
      "rewards/rejected": -0.19871282577514648,
      "step": 3480
    },
    {
      "epoch": 1.83,
      "learning_rate": 1.1280606857450387e-07,
      "logits/chosen": 0.5712449550628662,
      "logits/rejected": 0.6291993856430054,
      "logps/chosen": -243.8418731689453,
      "logps/rejected": -233.1319122314453,
      "loss": 1809.2799,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.11564090102910995,
      "rewards/margins": 0.11533119529485703,
      "rewards/rejected": -0.23097209632396698,
      "step": 3490
    },
    {
      "epoch": 1.83,
      "learning_rate": 1.0612098162470302e-07,
      "logits/chosen": 0.5486131906509399,
      "logits/rejected": 0.6034047603607178,
      "logps/chosen": -253.94577026367188,
      "logps/rejected": -243.55593872070312,
      "loss": 1937.2627,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10992947965860367,
      "rewards/margins": 0.09977956861257553,
      "rewards/rejected": -0.2097090482711792,
      "step": 3500
    },
    {
      "epoch": 1.83,
      "eval_logits/chosen": 0.5346845984458923,
      "eval_logits/rejected": 0.5903106927871704,
      "eval_logps/chosen": -269.0877990722656,
      "eval_logps/rejected": -255.17501831054688,
      "eval_loss": 2027.823974609375,
      "eval_rewards/accuracies": 0.6930000185966492,
      "eval_rewards/chosen": -0.12470405548810959,
      "eval_rewards/margins": 0.09155706316232681,
      "eval_rewards/rejected": -0.2162611186504364,
      "eval_runtime": 416.489,
      "eval_samples_per_second": 4.802,
      "eval_steps_per_second": 1.201,
      "step": 3500
    },
    {
      "epoch": 1.84,
      "learning_rate": 9.96357828840297e-08,
      "logits/chosen": 0.5791751742362976,
      "logits/rejected": 0.6535072326660156,
      "logps/chosen": -262.40301513671875,
      "logps/rejected": -260.7125549316406,
      "loss": 1964.6746,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1026170626282692,
      "rewards/margins": 0.08860354125499725,
      "rewards/rejected": -0.19122058153152466,
      "step": 3510
    },
    {
      "epoch": 1.84,
      "learning_rate": 9.335101386471285e-08,
      "logits/chosen": 0.5727615356445312,
      "logits/rejected": 0.5819220542907715,
      "logps/chosen": -284.75616455078125,
      "logps/rejected": -250.07083129882812,
      "loss": 2092.7152,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.12308394908905029,
      "rewards/margins": 0.08286546170711517,
      "rewards/rejected": -0.20594939589500427,
      "step": 3520
    },
    {
      "epoch": 1.85,
      "learning_rate": 8.726719934315648e-08,
      "logits/chosen": 0.5491209626197815,
      "logits/rejected": 0.5870348811149597,
      "logps/chosen": -249.99295043945312,
      "logps/rejected": -249.68679809570312,
      "loss": 1912.009,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10659299790859222,
      "rewards/margins": 0.09919796884059906,
      "rewards/rejected": -0.20579096674919128,
      "step": 3530
    },
    {
      "epoch": 1.85,
      "learning_rate": 8.138484731612273e-08,
      "logits/chosen": 0.6029896140098572,
      "logits/rejected": 0.6406581997871399,
      "logps/chosen": -256.2204284667969,
      "logps/rejected": -229.89990234375,
      "loss": 1913.6865,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10959267616271973,
      "rewards/margins": 0.1080545037984848,
      "rewards/rejected": -0.21764719486236572,
      "step": 3540
    },
    {
      "epoch": 1.86,
      "learning_rate": 7.57044489583128e-08,
      "logits/chosen": 0.5283448100090027,
      "logits/rejected": 0.5756082534790039,
      "logps/chosen": -266.2628479003906,
      "logps/rejected": -251.70962524414062,
      "loss": 2214.9162,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12659896910190582,
      "rewards/margins": 0.06956067681312561,
      "rewards/rejected": -0.19615966081619263,
      "step": 3550
    },
    {
      "epoch": 1.86,
      "learning_rate": 7.022647858135501e-08,
      "logits/chosen": 0.5648905038833618,
      "logits/rejected": 0.5851987600326538,
      "logps/chosen": -255.75704956054688,
      "logps/rejected": -235.11666870117188,
      "loss": 1911.8838,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10966980457305908,
      "rewards/margins": 0.09904567152261734,
      "rewards/rejected": -0.20871546864509583,
      "step": 3560
    },
    {
      "epoch": 1.87,
      "learning_rate": 6.495139359419922e-08,
      "logits/chosen": 0.5362564921379089,
      "logits/rejected": 0.629612922668457,
      "logps/chosen": -303.055419921875,
      "logps/rejected": -271.39288330078125,
      "loss": 1875.824,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1098160371184349,
      "rewards/margins": 0.11708301305770874,
      "rewards/rejected": -0.22689905762672424,
      "step": 3570
    },
    {
      "epoch": 1.87,
      "learning_rate": 5.987963446492384e-08,
      "logits/chosen": 0.5597736239433289,
      "logits/rejected": 0.5716227889060974,
      "logps/chosen": -262.5528564453125,
      "logps/rejected": -251.4736785888672,
      "loss": 1887.5355,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10405333340167999,
      "rewards/margins": 0.11675725132226944,
      "rewards/rejected": -0.22081057727336884,
      "step": 3580
    },
    {
      "epoch": 1.88,
      "learning_rate": 5.501162468395688e-08,
      "logits/chosen": 0.5817372798919678,
      "logits/rejected": 0.5784239768981934,
      "logps/chosen": -251.4989013671875,
      "logps/rejected": -250.26522827148438,
      "loss": 1920.6814,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12673336267471313,
      "rewards/margins": 0.11053230613470078,
      "rewards/rejected": -0.2372656762599945,
      "step": 3590
    },
    {
      "epoch": 1.88,
      "learning_rate": 5.034777072871394e-08,
      "logits/chosen": 0.5656172037124634,
      "logits/rejected": 0.6273232102394104,
      "logps/chosen": -250.90109252929688,
      "logps/rejected": -256.27178955078125,
      "loss": 2007.2062,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.1188703402876854,
      "rewards/margins": 0.09515853226184845,
      "rewards/rejected": -0.21402888000011444,
      "step": 3600
    },
    {
      "epoch": 1.88,
      "eval_logits/chosen": 0.5352125763893127,
      "eval_logits/rejected": 0.5910032391548157,
      "eval_logps/chosen": -269.0622863769531,
      "eval_logps/rejected": -255.18426513671875,
      "eval_loss": 2025.32275390625,
      "eval_rewards/accuracies": 0.6894999742507935,
      "eval_rewards/chosen": -0.12444862723350525,
      "eval_rewards/margins": 0.0919048860669136,
      "eval_rewards/rejected": -0.21635350584983826,
      "eval_runtime": 416.4513,
      "eval_samples_per_second": 4.802,
      "eval_steps_per_second": 1.201,
      "step": 3600
    },
    {
      "epoch": 1.89,
      "learning_rate": 4.5888462029658186e-08,
      "logits/chosen": 0.5575802326202393,
      "logits/rejected": 0.5975883603096008,
      "logps/chosen": -251.73623657226562,
      "logps/rejected": -250.5563201904297,
      "loss": 1952.7262,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.12859514355659485,
      "rewards/margins": 0.0945589691400528,
      "rewards/rejected": -0.22315411269664764,
      "step": 3610
    },
    {
      "epoch": 1.89,
      "learning_rate": 4.163407093778243e-08,
      "logits/chosen": 0.5034095048904419,
      "logits/rejected": 0.5554597973823547,
      "logps/chosen": -264.1334533691406,
      "logps/rejected": -260.69805908203125,
      "loss": 2102.8385,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.1385246217250824,
      "rewards/margins": 0.08275660127401352,
      "rewards/rejected": -0.22128121554851532,
      "step": 3620
    },
    {
      "epoch": 1.9,
      "learning_rate": 3.7584952693519025e-08,
      "logits/chosen": 0.5984662175178528,
      "logits/rejected": 0.5975054502487183,
      "logps/chosen": -270.71661376953125,
      "logps/rejected": -260.6340026855469,
      "loss": 1962.2367,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.11859796196222305,
      "rewards/margins": 0.0891033262014389,
      "rewards/rejected": -0.20770128071308136,
      "step": 3630
    },
    {
      "epoch": 1.91,
      "learning_rate": 3.37414453970758e-08,
      "logits/chosen": 0.5739267468452454,
      "logits/rejected": 0.577027440071106,
      "logps/chosen": -249.9311065673828,
      "logps/rejected": -226.3584442138672,
      "loss": 2134.9982,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.13262119889259338,
      "rewards/margins": 0.08364128321409225,
      "rewards/rejected": -0.21626248955726624,
      "step": 3640
    },
    {
      "epoch": 1.91,
      "learning_rate": 3.0103869980206145e-08,
      "logits/chosen": 0.6304140090942383,
      "logits/rejected": 0.63347989320755,
      "logps/chosen": -239.28915405273438,
      "logps/rejected": -258.83660888671875,
      "loss": 2049.8613,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.13652729988098145,
      "rewards/margins": 0.08251725137233734,
      "rewards/rejected": -0.2190445363521576,
      "step": 3650
    },
    {
      "epoch": 1.92,
      "learning_rate": 2.6672530179410183e-08,
      "logits/chosen": 0.5943226218223572,
      "logits/rejected": 0.6503596305847168,
      "logps/chosen": -264.0231018066406,
      "logps/rejected": -245.2039031982422,
      "loss": 2003.8398,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12203504145145416,
      "rewards/margins": 0.09293092787265778,
      "rewards/rejected": -0.21496596932411194,
      "step": 3660
    },
    {
      "epoch": 1.92,
      "learning_rate": 2.3447712510573928e-08,
      "logits/chosen": 0.6132981777191162,
      "logits/rejected": 0.6643080115318298,
      "logps/chosen": -258.1444091796875,
      "logps/rejected": -241.70986938476562,
      "loss": 1827.9928,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.12529726326465607,
      "rewards/margins": 0.11168257147073746,
      "rewards/rejected": -0.23697984218597412,
      "step": 3670
    },
    {
      "epoch": 1.93,
      "learning_rate": 2.04296862450451e-08,
      "logits/chosen": 0.5302231907844543,
      "logits/rejected": 0.5352843999862671,
      "logps/chosen": -270.13079833984375,
      "logps/rejected": -240.57931518554688,
      "loss": 2135.2258,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.10986328125,
      "rewards/margins": 0.07277282327413559,
      "rewards/rejected": -0.182636097073555,
      "step": 3680
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.7618703387147495e-08,
      "logits/chosen": 0.5543524622917175,
      "logits/rejected": 0.5519607663154602,
      "logps/chosen": -281.1496276855469,
      "logps/rejected": -274.3504943847656,
      "loss": 1951.3572,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10969813168048859,
      "rewards/margins": 0.09588075429201126,
      "rewards/rejected": -0.20557889342308044,
      "step": 3690
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.501499865314171e-08,
      "logits/chosen": 0.5777779817581177,
      "logits/rejected": 0.6056709289550781,
      "logps/chosen": -258.98333740234375,
      "logps/rejected": -245.7554473876953,
      "loss": 2076.715,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.11231188476085663,
      "rewards/margins": 0.09381435066461563,
      "rewards/rejected": -0.20612624287605286,
      "step": 3700
    },
    {
      "epoch": 1.94,
      "eval_logits/chosen": 0.5358251333236694,
      "eval_logits/rejected": 0.5913307666778564,
      "eval_logps/chosen": -269.0487365722656,
      "eval_logps/rejected": -255.13833618164062,
      "eval_loss": 2027.4857177734375,
      "eval_rewards/accuracies": 0.6919999718666077,
      "eval_rewards/chosen": -0.12431324273347855,
      "eval_rewards/margins": 0.09158134460449219,
      "eval_rewards/rejected": -0.21589456498622894,
      "eval_runtime": 416.7132,
      "eval_samples_per_second": 4.799,
      "eval_steps_per_second": 1.2,
      "step": 3700
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.2618789451623314e-08,
      "logits/chosen": 0.5645478963851929,
      "logits/rejected": 0.6109569072723389,
      "logps/chosen": -224.93197631835938,
      "logps/rejected": -236.52059936523438,
      "loss": 2014.5852,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.12038487195968628,
      "rewards/margins": 0.08836686611175537,
      "rewards/rejected": -0.20875172317028046,
      "step": 3710
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.0430275865371265e-08,
      "logits/chosen": 0.5508732795715332,
      "logits/rejected": 0.6115376353263855,
      "logps/chosen": -280.9700927734375,
      "logps/rejected": -276.9327087402344,
      "loss": 2099.5396,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12365047633647919,
      "rewards/margins": 0.08925069868564606,
      "rewards/rejected": -0.21290118992328644,
      "step": 3720
    },
    {
      "epoch": 1.95,
      "learning_rate": 8.449640634639878e-09,
      "logits/chosen": 0.5355272889137268,
      "logits/rejected": 0.5811390280723572,
      "logps/chosen": -234.78927612304688,
      "logps/rejected": -228.4997100830078,
      "loss": 2043.5014,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1230451837182045,
      "rewards/margins": 0.08070604503154755,
      "rewards/rejected": -0.20375123620033264,
      "step": 3730
    },
    {
      "epoch": 1.96,
      "learning_rate": 6.677049141901315e-09,
      "logits/chosen": 0.5882354974746704,
      "logits/rejected": 0.572884738445282,
      "logps/chosen": -238.9161834716797,
      "logps/rejected": -247.5298309326172,
      "loss": 2155.4453,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.13735826313495636,
      "rewards/margins": 0.06890521943569183,
      "rewards/rejected": -0.2062634974718094,
      "step": 3740
    },
    {
      "epoch": 1.96,
      "learning_rate": 5.112649398034686e-09,
      "logits/chosen": 0.6161108016967773,
      "logits/rejected": 0.6904915571212769,
      "logps/chosen": -284.59112548828125,
      "logps/rejected": -254.5317840576172,
      "loss": 2025.8148,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.125542551279068,
      "rewards/margins": 0.1034855991601944,
      "rewards/rejected": -0.2290281355381012,
      "step": 3750
    },
    {
      "epoch": 1.97,
      "learning_rate": 3.756572029968708e-09,
      "logits/chosen": 0.5819270610809326,
      "logits/rejected": 0.552914023399353,
      "logps/chosen": -255.23886108398438,
      "logps/rejected": -249.76144409179688,
      "loss": 1779.8414,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11097989976406097,
      "rewards/margins": 0.1095919981598854,
      "rewards/rejected": -0.22057190537452698,
      "step": 3760
    },
    {
      "epoch": 1.97,
      "learning_rate": 2.6089302697732133e-09,
      "logits/chosen": 0.5825963020324707,
      "logits/rejected": 0.5435997843742371,
      "logps/chosen": -250.9562225341797,
      "logps/rejected": -227.83010864257812,
      "loss": 1853.0563,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.11256451904773712,
      "rewards/margins": 0.10355798900127411,
      "rewards/rejected": -0.21612253785133362,
      "step": 3770
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.6698199452053199e-09,
      "logits/chosen": 0.6074197292327881,
      "logits/rejected": 0.6451854705810547,
      "logps/chosen": -269.6044616699219,
      "logps/rejected": -231.6178436279297,
      "loss": 1905.3814,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10596567392349243,
      "rewards/margins": 0.099820576608181,
      "rewards/rejected": -0.20578625798225403,
      "step": 3780
    },
    {
      "epoch": 1.98,
      "learning_rate": 9.393194717061127e-10,
      "logits/chosen": 0.5966477394104004,
      "logits/rejected": 0.57940673828125,
      "logps/chosen": -261.906982421875,
      "logps/rejected": -243.7214813232422,
      "loss": 2099.1896,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.12468767166137695,
      "rewards/margins": 0.0848483294248581,
      "rewards/rejected": -0.20953598618507385,
      "step": 3790
    },
    {
      "epoch": 1.99,
      "learning_rate": 4.1748984585560094e-10,
      "logits/chosen": 0.5209355354309082,
      "logits/rejected": 0.6011817455291748,
      "logps/chosen": -257.47882080078125,
      "logps/rejected": -253.18017578125,
      "loss": 2055.2201,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.12113461643457413,
      "rewards/margins": 0.09108567237854004,
      "rewards/rejected": -0.21222028136253357,
      "step": 3800
    },
    {
      "epoch": 1.99,
      "eval_logits/chosen": 0.5346859693527222,
      "eval_logits/rejected": 0.5902337431907654,
      "eval_logps/chosen": -269.0542907714844,
      "eval_logps/rejected": -255.1454620361328,
      "eval_loss": 2027.8082275390625,
      "eval_rewards/accuracies": 0.6919999718666077,
      "eval_rewards/chosen": -0.12436838448047638,
      "eval_rewards/margins": 0.09159712493419647,
      "eval_rewards/rejected": -0.21596547961235046,
      "eval_runtime": 416.5485,
      "eval_samples_per_second": 4.801,
      "eval_steps_per_second": 1.2,
      "step": 3800
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.0437464027707179e-10,
      "logits/chosen": 0.5776988863945007,
      "logits/rejected": 0.6123504638671875,
      "logps/chosen": -265.8362121582031,
      "logps/rejected": -237.8904571533203,
      "loss": 2055.609,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.1267283707857132,
      "rewards/margins": 0.08532574027776718,
      "rewards/rejected": -0.21205410361289978,
      "step": 3810
    },
    {
      "epoch": 2.0,
      "learning_rate": 0.0,
      "logits/chosen": 0.5118510127067566,
      "logits/rejected": 0.5845987200737,
      "logps/chosen": -274.1031188964844,
      "logps/rejected": -256.99688720703125,
      "loss": 2139.6068,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.13095693290233612,
      "rewards/margins": 0.07750894129276276,
      "rewards/rejected": -0.20846585929393768,
      "step": 3820
    },
    {
      "epoch": 2.0,
      "step": 3820,
      "total_flos": 0.0,
      "train_loss": 2099.8451463309884,
      "train_runtime": 42790.3459,
      "train_samples_per_second": 1.429,
      "train_steps_per_second": 0.089
    }
  ],
  "logging_steps": 10,
  "max_steps": 3820,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}