{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 478,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": -2.847970962524414,
      "logits/rejected": -2.79160213470459,
      "logps/chosen": -284.9612731933594,
      "logps/rejected": -276.45928955078125,
      "loss": 0.0866,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.754711151123047,
      "logits/rejected": -2.7527668476104736,
      "logps/chosen": -249.9619140625,
      "logps/rejected": -223.05694580078125,
      "loss": 0.0866,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00014172698138281703,
      "rewards/margins": -5.154276004759595e-05,
      "rewards/rejected": -9.018417040351778e-05,
      "step": 10
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.7448391914367676,
      "logits/rejected": -2.7453367710113525,
      "logps/chosen": -257.4018249511719,
      "logps/rejected": -247.55197143554688,
      "loss": 0.0866,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0005366196855902672,
      "rewards/margins": 0.0017497055232524872,
      "rewards/rejected": -0.0012130856048315763,
      "step": 20
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.8008646965026855,
      "logits/rejected": -2.753528356552124,
      "logps/chosen": -300.37725830078125,
      "logps/rejected": -261.90411376953125,
      "loss": 0.086,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0019977777265012264,
      "rewards/margins": 0.010120971128344536,
      "rewards/rejected": -0.008123192936182022,
      "step": 30
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.7638278007507324,
      "logits/rejected": -2.7516701221466064,
      "logps/chosen": -256.6244201660156,
      "logps/rejected": -274.86651611328125,
      "loss": 0.0851,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.00190549751278013,
      "rewards/margins": 0.025926370173692703,
      "rewards/rejected": -0.027831867337226868,
      "step": 40
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999733114418725e-07,
      "logits/chosen": -2.767770767211914,
      "logits/rejected": -2.7401928901672363,
      "logps/chosen": -284.50543212890625,
      "logps/rejected": -256.618408203125,
      "loss": 0.0834,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.02425984852015972,
      "rewards/margins": 0.06489187479019165,
      "rewards/rejected": -0.08915172517299652,
      "step": 50
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990398100856366e-07,
      "logits/chosen": -2.7361197471618652,
      "logits/rejected": -2.724640130996704,
      "logps/chosen": -282.0168151855469,
      "logps/rejected": -256.8324279785156,
      "loss": 0.0805,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06132340431213379,
      "rewards/margins": 0.10652206093072891,
      "rewards/rejected": -0.1678454577922821,
      "step": 60
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.967775735898179e-07,
      "logits/chosen": -2.7822699546813965,
      "logits/rejected": -2.739877223968506,
      "logps/chosen": -291.2421875,
      "logps/rejected": -274.023193359375,
      "loss": 0.0781,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.24106796085834503,
      "rewards/margins": 0.17975760996341705,
      "rewards/rejected": -0.4208255410194397,
      "step": 70
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.931986719649298e-07,
      "logits/chosen": -2.7829222679138184,
      "logits/rejected": -2.7552578449249268,
      "logps/chosen": -290.30157470703125,
      "logps/rejected": -332.9333801269531,
      "loss": 0.0744,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.29948586225509644,
      "rewards/margins": 0.3068729043006897,
      "rewards/rejected": -0.6063587665557861,
      "step": 80
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.883222001996351e-07,
      "logits/chosen": -2.8093197345733643,
      "logits/rejected": -2.7847156524658203,
      "logps/chosen": -308.68927001953125,
      "logps/rejected": -327.748046875,
      "loss": 0.0733,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3332245349884033,
      "rewards/margins": 0.4322693943977356,
      "rewards/rejected": -0.7654938697814941,
      "step": 90
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.821741763807186e-07,
      "logits/chosen": -2.776167392730713,
      "logits/rejected": -2.743318557739258,
      "logps/chosen": -356.5120544433594,
      "logps/rejected": -373.7995910644531,
      "loss": 0.0717,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.6754709482192993,
      "rewards/margins": 0.440376341342926,
      "rewards/rejected": -1.1158472299575806,
      "step": 100
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -2.728982448577881,
      "eval_logits/rejected": -2.712980031967163,
      "eval_logps/chosen": -316.240478515625,
      "eval_logps/rejected": -367.9753112792969,
      "eval_loss": 0.073273666203022,
      "eval_rewards/accuracies": 0.7265625,
      "eval_rewards/chosen": -0.5920084714889526,
      "eval_rewards/margins": 0.5142118334770203,
      "eval_rewards/rejected": -1.1062203645706177,
      "eval_runtime": 53.5143,
      "eval_samples_per_second": 37.373,
      "eval_steps_per_second": 0.598,
      "step": 100
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.747874028753375e-07,
      "logits/chosen": -2.6736016273498535,
      "logits/rejected": -2.6673741340637207,
      "logps/chosen": -319.99285888671875,
      "logps/rejected": -394.5204772949219,
      "loss": 0.0675,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.557295560836792,
      "rewards/margins": 0.660796582698822,
      "rewards/rejected": -1.2180922031402588,
      "step": 110
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.662012913161997e-07,
      "logits/chosen": -2.6141879558563232,
      "logits/rejected": -2.5749917030334473,
      "logps/chosen": -359.4350280761719,
      "logps/rejected": -392.105712890625,
      "loss": 0.0709,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.9189525842666626,
      "rewards/margins": 0.5283113718032837,
      "rewards/rejected": -1.4472639560699463,
      "step": 120
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5646165232345103e-07,
      "logits/chosen": -2.5792384147644043,
      "logits/rejected": -2.5439562797546387,
      "logps/chosen": -372.8755798339844,
      "logps/rejected": -363.8757629394531,
      "loss": 0.0701,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.8074409365653992,
      "rewards/margins": 0.42888420820236206,
      "rewards/rejected": -1.2363251447677612,
      "step": 130
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.456204510851956e-07,
      "logits/chosen": -2.4518723487854004,
      "logits/rejected": -2.429705858230591,
      "logps/chosen": -335.4825744628906,
      "logps/rejected": -358.73992919921875,
      "loss": 0.0687,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8712352514266968,
      "rewards/margins": 0.5185465812683105,
      "rewards/rejected": -1.3897819519042969,
      "step": 140
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.337355301007335e-07,
      "logits/chosen": -2.3888745307922363,
      "logits/rejected": -2.3675789833068848,
      "logps/chosen": -345.4061584472656,
      "logps/rejected": -378.5643005371094,
      "loss": 0.0674,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8017293810844421,
      "rewards/margins": 0.5842651128768921,
      "rewards/rejected": -1.385994553565979,
      "step": 150
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2087030056579986e-07,
      "logits/chosen": -2.339261531829834,
      "logits/rejected": -2.3114781379699707,
      "logps/chosen": -338.3687744140625,
      "logps/rejected": -375.3154296875,
      "loss": 0.0684,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.7738892436027527,
      "rewards/margins": 0.6639295816421509,
      "rewards/rejected": -1.4378188848495483,
      "step": 160
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.070934040463998e-07,
      "logits/chosen": -2.2538902759552,
      "logits/rejected": -2.2150468826293945,
      "logps/chosen": -380.0686950683594,
      "logps/rejected": -427.46514892578125,
      "loss": 0.0679,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.1141223907470703,
      "rewards/margins": 0.6542497873306274,
      "rewards/rejected": -1.7683721780776978,
      "step": 170
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.9247834624635404e-07,
      "logits/chosen": -2.331003427505493,
      "logits/rejected": -2.2696776390075684,
      "logps/chosen": -374.4627380371094,
      "logps/rejected": -422.1844177246094,
      "loss": 0.066,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.980134129524231,
      "rewards/margins": 0.7622623443603516,
      "rewards/rejected": -1.742396593093872,
      "step": 180
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7710310482256523e-07,
      "logits/chosen": -2.2278285026550293,
      "logits/rejected": -2.187769651412964,
      "logps/chosen": -367.53619384765625,
      "logps/rejected": -409.8011169433594,
      "loss": 0.0653,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.1032383441925049,
      "rewards/margins": 0.6265621781349182,
      "rewards/rejected": -1.7298005819320679,
      "step": 190
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.610497133404795e-07,
      "logits/chosen": -2.2965497970581055,
      "logits/rejected": -2.2077629566192627,
      "logps/chosen": -437.07275390625,
      "logps/rejected": -473.34228515625,
      "loss": 0.0672,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.024962067604065,
      "rewards/margins": 0.9094398617744446,
      "rewards/rejected": -1.9344017505645752,
      "step": 200
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": -2.23770809173584,
      "eval_logits/rejected": -2.204430103302002,
      "eval_logps/chosen": -350.147216796875,
      "eval_logps/rejected": -429.3445129394531,
      "eval_loss": 0.06620719283819199,
      "eval_rewards/accuracies": 0.7421875,
      "eval_rewards/chosen": -0.9310759902000427,
      "eval_rewards/margins": 0.7888363599777222,
      "eval_rewards/rejected": -1.7199124097824097,
      "eval_runtime": 53.5039,
      "eval_samples_per_second": 37.38,
      "eval_steps_per_second": 0.598,
      "step": 200
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4440382358952115e-07,
      "logits/chosen": -2.2368922233581543,
      "logits/rejected": -2.164578437805176,
      "logps/chosen": -367.9010009765625,
      "logps/rejected": -403.06689453125,
      "loss": 0.0688,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.0069431066513062,
      "rewards/margins": 0.6194397211074829,
      "rewards/rejected": -1.6263830661773682,
      "step": 210
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": -2.223553419113159,
      "logits/rejected": -2.1863625049591064,
      "logps/chosen": -383.9325256347656,
      "logps/rejected": -408.54156494140625,
      "loss": 0.066,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.893720805644989,
      "rewards/margins": 0.7813506126403809,
      "rewards/rejected": -1.6750714778900146,
      "step": 220
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.096924887558854e-07,
      "logits/chosen": -2.1857941150665283,
      "logits/rejected": -2.1605677604675293,
      "logps/chosen": -385.0683288574219,
      "logps/rejected": -465.76214599609375,
      "loss": 0.065,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0325820446014404,
      "rewards/margins": 0.7163792848587036,
      "rewards/rejected": -1.7489612102508545,
      "step": 230
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9181224366319943e-07,
      "logits/chosen": -2.1450023651123047,
      "logits/rejected": -2.1016509532928467,
      "logps/chosen": -375.78448486328125,
      "logps/rejected": -456.2076110839844,
      "loss": 0.0663,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.2208166122436523,
      "rewards/margins": 0.755986213684082,
      "rewards/rejected": -1.9768028259277344,
      "step": 240
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7370891215954565e-07,
      "logits/chosen": -2.084066390991211,
      "logits/rejected": -2.000739574432373,
      "logps/chosen": -408.6034240722656,
      "logps/rejected": -460.944091796875,
      "loss": 0.0656,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.496781587600708,
      "rewards/margins": 0.7584124207496643,
      "rewards/rejected": -2.2551939487457275,
      "step": 250
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.55479083351317e-07,
      "logits/chosen": -2.1052582263946533,
      "logits/rejected": -2.0781798362731934,
      "logps/chosen": -385.717529296875,
      "logps/rejected": -427.49053955078125,
      "loss": 0.0675,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.171087622642517,
      "rewards/margins": 0.6191247701644897,
      "rewards/rejected": -1.7902123928070068,
      "step": 260
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3722002126275822e-07,
      "logits/chosen": -2.060842514038086,
      "logits/rejected": -2.032809257507324,
      "logps/chosen": -411.20904541015625,
      "logps/rejected": -466.87945556640625,
      "loss": 0.0644,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.3635554313659668,
      "rewards/margins": 0.7496285438537598,
      "rewards/rejected": -2.1131839752197266,
      "step": 270
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.19029145890313e-07,
      "logits/chosen": -2.0691192150115967,
      "logits/rejected": -2.014458417892456,
      "logps/chosen": -368.3045349121094,
      "logps/rejected": -434.04840087890625,
      "loss": 0.0659,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.2689889669418335,
      "rewards/margins": 0.7694240808486938,
      "rewards/rejected": -2.0384132862091064,
      "step": 280
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0100351342479216e-07,
      "logits/chosen": -2.0620741844177246,
      "logits/rejected": -2.0015225410461426,
      "logps/chosen": -380.07489013671875,
      "logps/rejected": -437.5985412597656,
      "loss": 0.0641,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.3346366882324219,
      "rewards/margins": 0.6989216208457947,
      "rewards/rejected": -2.0335583686828613,
      "step": 290
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8323929841460178e-07,
      "logits/chosen": -2.0831615924835205,
      "logits/rejected": -2.0331509113311768,
      "logps/chosen": -385.72332763671875,
      "logps/rejected": -458.3067932128906,
      "loss": 0.0648,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.294547200202942,
      "rewards/margins": 0.7150087356567383,
      "rewards/rejected": -2.0095558166503906,
      "step": 300
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -2.1098170280456543,
      "eval_logits/rejected": -2.072664976119995,
      "eval_logps/chosen": -382.6705322265625,
      "eval_logps/rejected": -471.1217041015625,
      "eval_loss": 0.06434247642755508,
      "eval_rewards/accuracies": 0.7734375,
      "eval_rewards/chosen": -1.256308674812317,
      "eval_rewards/margins": 0.8813759088516235,
      "eval_rewards/rejected": -2.1376843452453613,
      "eval_runtime": 53.4979,
      "eval_samples_per_second": 37.385,
      "eval_steps_per_second": 0.598,
      "step": 300
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6583128063291573e-07,
      "logits/chosen": -2.038897752761841,
      "logits/rejected": -1.9522100687026978,
      "logps/chosen": -395.3385009765625,
      "logps/rejected": -463.807861328125,
      "loss": 0.0619,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.2215960025787354,
      "rewards/margins": 0.9447793960571289,
      "rewards/rejected": -2.1663756370544434,
      "step": 310
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.488723393865766e-07,
      "logits/chosen": -2.0476739406585693,
      "logits/rejected": -2.003208875656128,
      "logps/chosen": -440.6886291503906,
      "logps/rejected": -456.0061950683594,
      "loss": 0.0649,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.443062663078308,
      "rewards/margins": 0.7629345655441284,
      "rewards/rejected": -2.2059974670410156,
      "step": 320
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3245295796480788e-07,
      "logits/chosen": -2.0648202896118164,
      "logits/rejected": -2.0035393238067627,
      "logps/chosen": -399.42724609375,
      "logps/rejected": -452.73468017578125,
      "loss": 0.0643,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.472140908241272,
      "rewards/margins": 0.7188968062400818,
      "rewards/rejected": -2.191037654876709,
      "step": 330
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1666074087171627e-07,
      "logits/chosen": -2.010235548019409,
      "logits/rejected": -1.9665842056274414,
      "logps/chosen": -401.20391845703125,
      "logps/rejected": -479.63140869140625,
      "loss": 0.0655,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.4996058940887451,
      "rewards/margins": 0.8774793744087219,
      "rewards/rejected": -2.3770852088928223,
      "step": 340
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0157994641835734e-07,
      "logits/chosen": -1.9906257390975952,
      "logits/rejected": -1.9489715099334717,
      "logps/chosen": -377.80291748046875,
      "logps/rejected": -473.23565673828125,
      "loss": 0.0621,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.5185272693634033,
      "rewards/margins": 0.806858241558075,
      "rewards/rejected": -2.325385570526123,
      "step": 350
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.729103716819111e-08,
      "logits/chosen": -2.0000805854797363,
      "logits/rejected": -1.9243812561035156,
      "logps/chosen": -406.80120849609375,
      "logps/rejected": -481.9264221191406,
      "loss": 0.0596,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3953304290771484,
      "rewards/margins": 1.0305477380752563,
      "rewards/rejected": -2.4258780479431152,
      "step": 360
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.387025063449081e-08,
      "logits/chosen": -1.9636814594268799,
      "logits/rejected": -1.941389799118042,
      "logps/chosen": -379.83538818359375,
      "logps/rejected": -478.5880432128906,
      "loss": 0.0611,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.2458791732788086,
      "rewards/margins": 0.9973210096359253,
      "rewards/rejected": -2.2432003021240234,
      "step": 370
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.138919252022435e-08,
      "logits/chosen": -2.004906415939331,
      "logits/rejected": -1.9661741256713867,
      "logps/chosen": -411.92626953125,
      "logps/rejected": -458.4734802246094,
      "loss": 0.0625,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.46096932888031,
      "rewards/margins": 0.755499005317688,
      "rewards/rejected": -2.216468334197998,
      "step": 380
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.991445467064689e-08,
      "logits/chosen": -1.9821668863296509,
      "logits/rejected": -1.9519681930541992,
      "logps/chosen": -385.4888000488281,
      "logps/rejected": -463.5560607910156,
      "loss": 0.0612,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.282893419265747,
      "rewards/margins": 0.933008074760437,
      "rewards/rejected": -2.2159013748168945,
      "step": 390
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9507259776993954e-08,
      "logits/chosen": -2.0385870933532715,
      "logits/rejected": -2.0015995502471924,
      "logps/chosen": -436.51800537109375,
      "logps/rejected": -499.8250427246094,
      "loss": 0.0636,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.3635799884796143,
      "rewards/margins": 0.9055550694465637,
      "rewards/rejected": -2.269134998321533,
      "step": 400
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -2.047151565551758,
      "eval_logits/rejected": -2.0084266662597656,
      "eval_logps/chosen": -391.10418701171875,
      "eval_logps/rejected": -488.8221740722656,
      "eval_loss": 0.06317802518606186,
      "eval_rewards/accuracies": 0.7734375,
      "eval_rewards/chosen": -1.3406453132629395,
      "eval_rewards/margins": 0.9740438461303711,
      "eval_rewards/rejected": -2.3146889209747314,
      "eval_runtime": 53.4405,
      "eval_samples_per_second": 37.425,
      "eval_steps_per_second": 0.599,
      "step": 400
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.022313472693447e-08,
      "logits/chosen": -2.0089824199676514,
      "logits/rejected": -1.990161657333374,
      "logps/chosen": -397.4513854980469,
      "logps/rejected": -437.2802734375,
      "loss": 0.064,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4276202917099,
      "rewards/margins": 0.6205279231071472,
      "rewards/rejected": -2.0481481552124023,
      "step": 410
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2111614344599684e-08,
      "logits/chosen": -2.084846019744873,
      "logits/rejected": -2.0091195106506348,
      "logps/chosen": -399.70892333984375,
      "logps/rejected": -477.39215087890625,
      "loss": 0.061,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2201029062271118,
      "rewards/margins": 1.0571900606155396,
      "rewards/rejected": -2.2772929668426514,
      "step": 420
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.521597710086439e-08,
      "logits/chosen": -2.076000928878784,
      "logits/rejected": -2.0238547325134277,
      "logps/chosen": -406.68243408203125,
      "logps/rejected": -459.352783203125,
      "loss": 0.0611,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.3172928094863892,
      "rewards/margins": 0.819710910320282,
      "rewards/rejected": -2.1370038986206055,
      "step": 430
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.57301420397924e-09,
      "logits/chosen": -1.9988025426864624,
      "logits/rejected": -1.979654312133789,
      "logps/chosen": -415.54779052734375,
      "logps/rejected": -451.9610900878906,
      "loss": 0.0628,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.372821569442749,
      "rewards/margins": 0.7440255880355835,
      "rewards/rejected": -2.116847276687622,
      "step": 440
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.212833302556258e-09,
      "logits/chosen": -2.028916120529175,
      "logits/rejected": -1.9799381494522095,
      "logps/chosen": -409.3075866699219,
      "logps/rejected": -491.86444091796875,
      "loss": 0.0605,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.326899766921997,
      "rewards/margins": 0.9430659413337708,
      "rewards/rejected": -2.269965648651123,
      "step": 450
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.158697848236607e-09,
      "logits/chosen": -1.9905271530151367,
      "logits/rejected": -1.9591131210327148,
      "logps/chosen": -391.362060546875,
      "logps/rejected": -461.91204833984375,
      "loss": 0.0604,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.2606008052825928,
      "rewards/margins": 0.8890584111213684,
      "rewards/rejected": -2.1496591567993164,
      "step": 460
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.269029751107489e-10,
      "logits/chosen": -2.0331690311431885,
      "logits/rejected": -1.9691358804702759,
      "logps/chosen": -417.90203857421875,
      "logps/rejected": -465.673828125,
      "loss": 0.0625,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.3203939199447632,
      "rewards/margins": 0.9791151285171509,
      "rewards/rejected": -2.299509048461914,
      "step": 470
    },
    {
      "epoch": 1.0,
      "step": 478,
      "total_flos": 0.0,
      "train_loss": 0.06801126423490596,
      "train_runtime": 3957.1126,
      "train_samples_per_second": 15.449,
      "train_steps_per_second": 0.121
    }
  ],
  "logging_steps": 10,
  "max_steps": 478,
  "num_train_epochs": 1,
  "save_steps": 1000,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}