{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 9.99018645731109,
  "eval_steps": 100,
  "global_step": 5090,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.001962708537782139,
      "grad_norm": 2.4117076017287205,
      "learning_rate": 9.803921568627451e-09,
      "logits/chosen": -1.125,
      "logits/rejected": -1.1875,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -14.5,
      "logps/chosen_top_tokens": -0.0005645751953125,
      "logps/rejected": -520.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00054168701171875,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.019627085377821395,
      "grad_norm": 2.3800058601187866,
      "learning_rate": 9.80392156862745e-08,
      "logits/chosen": -1.1640625,
      "logits/rejected": -1.203125,
      "logps/chosen": -380.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000804901123046875,
      "logps/rejected": -316.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.000827789306640625,
      "loss": 0.6922,
      "rewards/accuracies": 0.41111111640930176,
      "rewards/chosen": -0.000202178955078125,
      "rewards/margins": 0.0035247802734375,
      "rewards/rejected": -0.00372314453125,
      "step": 10
    },
    {
      "epoch": 0.03925417075564279,
      "grad_norm": 2.4064882227881057,
      "learning_rate": 1.96078431372549e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -1.1484375,
      "logps/chosen": -374.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000835418701171875,
      "logps/rejected": -324.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00084686279296875,
      "loss": 0.6913,
      "rewards/accuracies": 0.4399999976158142,
      "rewards/chosen": 0.003662109375,
      "rewards/margins": 0.0033111572265625,
      "rewards/rejected": 0.0003528594970703125,
      "step": 20
    },
    {
      "epoch": 0.058881256133464184,
      "grad_norm": 2.3536995350535426,
      "learning_rate": 2.941176470588235e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -1.1796875,
      "logps/chosen": -364.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000762939453125,
      "logps/rejected": -324.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.00078582763671875,
      "loss": 0.6938,
      "rewards/accuracies": 0.3850000202655792,
      "rewards/chosen": -0.0030670166015625,
      "rewards/margins": -0.0067138671875,
      "rewards/rejected": 0.003631591796875,
      "step": 30
    },
    {
      "epoch": 0.07850834151128558,
      "grad_norm": 2.3870217018270155,
      "learning_rate": 3.92156862745098e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -1.15625,
      "logps/chosen": -378.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000759124755859375,
      "logps/rejected": -338.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.000804901123046875,
      "loss": 0.6929,
      "rewards/accuracies": 0.445000022649765,
      "rewards/chosen": 0.000881195068359375,
      "rewards/margins": 0.00244140625,
      "rewards/rejected": -0.00154876708984375,
      "step": 40
    },
    {
      "epoch": 0.09813542688910697,
      "grad_norm": 2.4788478916800147,
      "learning_rate": 4.901960784313725e-07,
      "logits/chosen": -1.1171875,
      "logits/rejected": -1.1484375,
      "logps/chosen": -406.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000774383544921875,
      "logps/rejected": -352.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00080108642578125,
      "loss": 0.6915,
      "rewards/accuracies": 0.5400000214576721,
      "rewards/chosen": 0.0026397705078125,
      "rewards/margins": 0.005889892578125,
      "rewards/rejected": -0.00323486328125,
      "step": 50
    },
    {
      "epoch": 0.11776251226692837,
      "grad_norm": 2.360316334548125,
      "learning_rate": 4.995237599803335e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -1.203125,
      "logps/chosen": -406.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000782012939453125,
      "logps/rejected": -322.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.0008087158203125,
      "loss": 0.6913,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.00244140625,
      "rewards/margins": 0.00201416015625,
      "rewards/rejected": 0.000431060791015625,
      "step": 60
    },
    {
      "epoch": 0.13738959764474976,
      "grad_norm": 2.3051434353276847,
      "learning_rate": 4.978798275112142e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.1328125,
      "logps/chosen": -372.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00078582763671875,
      "logps/rejected": -330.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000789642333984375,
      "loss": 0.688,
      "rewards/accuracies": 0.5049999952316284,
      "rewards/chosen": 0.00897216796875,
      "rewards/margins": 0.01190185546875,
      "rewards/rejected": -0.0028839111328125,
      "step": 70
    },
    {
      "epoch": 0.15701668302257116,
      "grad_norm": 2.2866846976386,
      "learning_rate": 4.950700530747689e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.1484375,
      "logps/chosen": -378.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000934600830078125,
      "logps/rejected": -308.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00087738037109375,
      "loss": 0.685,
      "rewards/accuracies": 0.5450000166893005,
      "rewards/chosen": -0.00121307373046875,
      "rewards/margins": 0.01483154296875,
      "rewards/rejected": -0.01611328125,
      "step": 80
    },
    {
      "epoch": 0.17664376840039253,
      "grad_norm": 2.3053347338418098,
      "learning_rate": 4.911076517558622e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -1.15625,
      "logps/chosen": -382.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000823974609375,
      "logps/rejected": -346.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00084686279296875,
      "loss": 0.6832,
      "rewards/accuracies": 0.5600000023841858,
      "rewards/chosen": -0.0106201171875,
      "rewards/margins": 0.0159912109375,
      "rewards/rejected": -0.026611328125,
      "step": 90
    },
    {
      "epoch": 0.19627085377821393,
      "grad_norm": 2.2125416576513732,
      "learning_rate": 4.860112597371772e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -1.171875,
      "logps/chosen": -372.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000904083251953125,
      "logps/rejected": -328.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.678,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.017333984375,
      "rewards/margins": 0.0361328125,
      "rewards/rejected": -0.053466796875,
      "step": 100
    },
    {
      "epoch": 0.19627085377821393,
      "eval_logits/chosen": -1.09375,
      "eval_logits/rejected": -1.15625,
      "eval_logps/chosen": -396.0,
      "eval_logps/chosen_bottom_tokens": -14.0625,
      "eval_logps/chosen_top_tokens": -0.0008697509765625,
      "eval_logps/rejected": -344.0,
      "eval_logps/rejected_bottom_tokens": -14.0,
      "eval_logps/rejected_top_tokens": -0.0008697509765625,
      "eval_loss": 0.6789160370826721,
      "eval_rewards/accuracies": 0.5880597233772278,
      "eval_rewards/chosen": -0.0274658203125,
      "eval_rewards/margins": 0.033203125,
      "eval_rewards/rejected": -0.060791015625,
      "eval_runtime": 111.5869,
      "eval_samples_per_second": 17.923,
      "eval_steps_per_second": 0.6,
      "step": 100
    },
    {
      "epoch": 0.21589793915603533,
      "grad_norm": 2.438395616681449,
      "learning_rate": 4.798048466485017e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.109375,
      "logps/chosen": -344.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000835418701171875,
      "logps/rejected": -332.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000873565673828125,
      "loss": 0.6804,
      "rewards/accuracies": 0.5899999737739563,
      "rewards/chosen": -0.037109375,
      "rewards/margins": 0.02001953125,
      "rewards/rejected": -0.05712890625,
      "step": 110
    },
    {
      "epoch": 0.23552502453385674,
      "grad_norm": 2.226213549318803,
      "learning_rate": 4.725176028314541e-07,
      "logits/chosen": -1.109375,
      "logits/rejected": -1.1171875,
      "logps/chosen": -372.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.0008544921875,
      "logps/rejected": -354.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.0008544921875,
      "loss": 0.6745,
      "rewards/accuracies": 0.6399999856948853,
      "rewards/chosen": -0.03564453125,
      "rewards/margins": 0.0517578125,
      "rewards/rejected": -0.08740234375,
      "step": 120
    },
    {
      "epoch": 0.25515210991167814,
      "grad_norm": 2.4135162897156706,
      "learning_rate": 4.641838020498713e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.1640625,
      "logps/chosen": -408.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000934600830078125,
      "logps/rejected": -338.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.00092315673828125,
      "loss": 0.6674,
      "rewards/accuracies": 0.5849999785423279,
      "rewards/chosen": -0.0703125,
      "rewards/margins": 0.0517578125,
      "rewards/rejected": -0.1220703125,
      "step": 130
    },
    {
      "epoch": 0.2747791952894995,
      "grad_norm": 2.4502181786024004,
      "learning_rate": 4.5484264029156733e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.1484375,
      "logps/chosen": -386.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000812530517578125,
      "logps/rejected": -336.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.00083160400390625,
      "loss": 0.6635,
      "rewards/accuracies": 0.5900000333786011,
      "rewards/chosen": -0.1015625,
      "rewards/margins": 0.048828125,
      "rewards/rejected": -0.150390625,
      "step": 140
    },
    {
      "epoch": 0.2944062806673209,
      "grad_norm": 2.4663119079457614,
      "learning_rate": 4.445380514196192e-07,
      "logits/chosen": -1.09375,
      "logits/rejected": -1.171875,
      "logps/chosen": -428.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.00087738037109375,
      "logps/rejected": -356.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.000858306884765625,
      "loss": 0.668,
      "rewards/accuracies": 0.6350000500679016,
      "rewards/chosen": -0.12890625,
      "rewards/margins": 0.0673828125,
      "rewards/rejected": -0.1962890625,
      "step": 150
    },
    {
      "epoch": 0.3140333660451423,
      "grad_norm": 2.455591342132379,
      "learning_rate": 4.33318500540218e-07,
      "logits/chosen": -1.0859375,
      "logits/rejected": -1.1328125,
      "logps/chosen": -408.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.00089263916015625,
      "logps/rejected": -368.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000904083251953125,
      "loss": 0.6655,
      "rewards/accuracies": 0.6050000190734863,
      "rewards/chosen": -0.16796875,
      "rewards/margins": 0.078125,
      "rewards/rejected": -0.24609375,
      "step": 160
    },
    {
      "epoch": 0.3336604514229637,
      "grad_norm": 2.436300399124971,
      "learning_rate": 4.2123675605892985e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.1484375,
      "logps/chosen": -422.0,
      "logps/chosen_bottom_tokens": -14.3125,
      "logps/chosen_top_tokens": -0.00101470947265625,
      "logps/rejected": -364.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.00098419189453125,
      "loss": 0.6585,
      "rewards/accuracies": 0.6450000405311584,
      "rewards/chosen": -0.1826171875,
      "rewards/margins": 0.115234375,
      "rewards/rejected": -0.296875,
      "step": 170
    },
    {
      "epoch": 0.35328753680078506,
      "grad_norm": 2.5546008416763035,
      "learning_rate": 4.0834964149744333e-07,
      "logits/chosen": -1.1015625,
      "logits/rejected": -1.15625,
      "logps/chosen": -416.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.00093841552734375,
      "logps/rejected": -380.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.00099945068359375,
      "loss": 0.6643,
      "rewards/accuracies": 0.6100000143051147,
      "rewards/chosen": -0.232421875,
      "rewards/margins": 0.06396484375,
      "rewards/rejected": -0.296875,
      "step": 180
    },
    {
      "epoch": 0.3729146221786065,
      "grad_norm": 2.555290762655567,
      "learning_rate": 3.947177682380738e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.203125,
      "logps/chosen": -378.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000789642333984375,
      "logps/rejected": -356.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.000823974609375,
      "loss": 0.6499,
      "rewards/accuracies": 0.6450001001358032,
      "rewards/chosen": -0.2412109375,
      "rewards/margins": 0.11669921875,
      "rewards/rejected": -0.357421875,
      "step": 190
    },
    {
      "epoch": 0.39254170755642787,
      "grad_norm": 2.7737043586573313,
      "learning_rate": 3.804052504529933e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.2265625,
      "logps/chosen": -392.0,
      "logps/chosen_bottom_tokens": -14.375,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -370.0,
      "logps/rejected_bottom_tokens": -14.375,
      "logps/rejected_top_tokens": -0.00087738037109375,
      "loss": 0.645,
      "rewards/accuracies": 0.6350000500679016,
      "rewards/chosen": -0.271484375,
      "rewards/margins": 0.1396484375,
      "rewards/rejected": -0.41015625,
      "step": 200
    },
    {
      "epoch": 0.39254170755642787,
      "eval_logits/chosen": -1.15625,
      "eval_logits/rejected": -1.203125,
      "eval_logps/chosen": -422.0,
      "eval_logps/chosen_bottom_tokens": -14.375,
      "eval_logps/chosen_top_tokens": -0.000911712646484375,
      "eval_logps/rejected": -380.0,
      "eval_logps/rejected_bottom_tokens": -14.3125,
      "eval_logps/rejected_top_tokens": -0.000919342041015625,
      "eval_loss": 0.6488671898841858,
      "eval_rewards/accuracies": 0.6447761058807373,
      "eval_rewards/chosen": -0.287109375,
      "eval_rewards/margins": 0.13671875,
      "eval_rewards/rejected": -0.423828125,
      "eval_runtime": 111.5112,
      "eval_samples_per_second": 17.935,
      "eval_steps_per_second": 0.601,
      "step": 200
    },
    {
      "epoch": 0.41216879293424924,
      "grad_norm": 2.8286672144445277,
      "learning_rate": 3.654794035589483e-07,
      "logits/chosen": -1.1328125,
      "logits/rejected": -1.1640625,
      "logps/chosen": -362.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.00091552734375,
      "logps/rejected": -344.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.000926971435546875,
      "loss": 0.6512,
      "rewards/accuracies": 0.6149999499320984,
      "rewards/chosen": -0.298828125,
      "rewards/margins": 0.12060546875,
      "rewards/rejected": -0.419921875,
      "step": 210
    },
    {
      "epoch": 0.43179587831207067,
      "grad_norm": 2.98579141751378,
      "learning_rate": 3.5001042761570826e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -1.2109375,
      "logps/chosen": -414.0,
      "logps/chosen_bottom_tokens": -14.5,
      "logps/chosen_top_tokens": -0.000762939453125,
      "logps/rejected": -398.0,
      "logps/rejected_bottom_tokens": -14.375,
      "logps/rejected_top_tokens": -0.000743865966796875,
      "loss": 0.6507,
      "rewards/accuracies": 0.5800000429153442,
      "rewards/chosen": -0.333984375,
      "rewards/margins": 0.11279296875,
      "rewards/rejected": -0.447265625,
      "step": 220
    },
    {
      "epoch": 0.45142296368989204,
      "grad_norm": 2.849801650804548,
      "learning_rate": 3.34071077157304e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -1.2265625,
      "logps/chosen": -388.0,
      "logps/chosen_bottom_tokens": -14.375,
      "logps/chosen_top_tokens": -0.00075531005859375,
      "logps/rejected": -354.0,
      "logps/rejected_bottom_tokens": -14.3125,
      "logps/rejected_top_tokens": -0.000827789306640625,
      "loss": 0.6464,
      "rewards/accuracies": 0.6299999952316284,
      "rewards/chosen": -0.353515625,
      "rewards/margins": 0.1337890625,
      "rewards/rejected": -0.48828125,
      "step": 230
    },
    {
      "epoch": 0.47105004906771347,
      "grad_norm": 3.020709895469043,
      "learning_rate": 3.1773631900892204e-07,
      "logits/chosen": -1.1484375,
      "logits/rejected": -1.1875,
      "logps/chosen": -416.0,
      "logps/chosen_bottom_tokens": -14.375,
      "logps/chosen_top_tokens": -0.000759124755859375,
      "logps/rejected": -396.0,
      "logps/rejected_bottom_tokens": -14.375,
      "logps/rejected_top_tokens": -0.0007781982421875,
      "loss": 0.6442,
      "rewards/accuracies": 0.6200000047683716,
      "rewards/chosen": -0.38671875,
      "rewards/margins": 0.1337890625,
      "rewards/rejected": -0.51953125,
      "step": 240
    },
    {
      "epoch": 0.49067713444553485,
      "grad_norm": 2.801068325901482,
      "learning_rate": 3.0108297969883103e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -1.1953125,
      "logps/chosen": -426.0,
      "logps/chosen_bottom_tokens": -14.4375,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -398.0,
      "logps/rejected_bottom_tokens": -14.375,
      "logps/rejected_top_tokens": -0.00081634521484375,
      "loss": 0.6347,
      "rewards/accuracies": 0.64000004529953,
      "rewards/chosen": -0.39453125,
      "rewards/margins": 0.1865234375,
      "rewards/rejected": -0.58203125,
      "step": 250
    },
    {
      "epoch": 0.5103042198233563,
      "grad_norm": 2.8119914001202835,
      "learning_rate": 2.8418938412365013e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -1.203125,
      "logps/chosen": -396.0,
      "logps/chosen_bottom_tokens": -14.3125,
      "logps/chosen_top_tokens": -0.000865936279296875,
      "logps/rejected": -372.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.000885009765625,
      "loss": 0.6381,
      "rewards/accuracies": 0.6350000500679016,
      "rewards/chosen": -0.373046875,
      "rewards/margins": 0.171875,
      "rewards/rejected": -0.546875,
      "step": 260
    },
    {
      "epoch": 0.5299313052011776,
      "grad_norm": 2.914608701481186,
      "learning_rate": 2.671349871664101e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -1.171875,
      "logps/chosen": -398.0,
      "logps/chosen_bottom_tokens": -14.3125,
      "logps/chosen_top_tokens": -0.000782012939453125,
      "logps/rejected": -386.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.00077056884765625,
      "loss": 0.6315,
      "rewards/accuracies": 0.64000004529953,
      "rewards/chosen": -0.40625,
      "rewards/margins": 0.17578125,
      "rewards/rejected": -0.58203125,
      "step": 270
    },
    {
      "epoch": 0.549558390578999,
      "grad_norm": 2.974677635397429,
      "learning_rate": 2.5e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -1.1953125,
      "logps/chosen": -438.0,
      "logps/chosen_bottom_tokens": -14.3125,
      "logps/chosen_top_tokens": -0.000911712646484375,
      "logps/rejected": -402.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.6384,
      "rewards/accuracies": 0.6600000262260437,
      "rewards/chosen": -0.42578125,
      "rewards/margins": 0.21484375,
      "rewards/rejected": -0.640625,
      "step": 280
    },
    {
      "epoch": 0.5691854759568205,
      "grad_norm": 3.4767790428686234,
      "learning_rate": 2.3286501283358982e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.2421875,
      "logps/chosen": -412.0,
      "logps/chosen_bottom_tokens": -14.3125,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -376.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.000885009765625,
      "loss": 0.632,
      "rewards/accuracies": 0.5750000476837158,
      "rewards/chosen": -0.4609375,
      "rewards/margins": 0.16015625,
      "rewards/rejected": -0.62109375,
      "step": 290
    },
    {
      "epoch": 0.5888125613346418,
      "grad_norm": 3.0983859451271565,
      "learning_rate": 2.1581061587634987e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -1.2421875,
      "logps/chosen": -428.0,
      "logps/chosen_bottom_tokens": -14.3125,
      "logps/chosen_top_tokens": -0.00075531005859375,
      "logps/rejected": -388.0,
      "logps/rejected_bottom_tokens": -14.3125,
      "logps/rejected_top_tokens": -0.00079345703125,
      "loss": 0.6396,
      "rewards/accuracies": 0.5999999642372131,
      "rewards/chosen": -0.482421875,
      "rewards/margins": 0.162109375,
      "rewards/rejected": -0.64453125,
      "step": 300
    },
    {
      "epoch": 0.5888125613346418,
      "eval_logits/chosen": -1.1875,
      "eval_logits/rejected": -1.234375,
      "eval_logps/chosen": -438.0,
      "eval_logps/chosen_bottom_tokens": -14.375,
      "eval_logps/chosen_top_tokens": -0.0007476806640625,
      "eval_logps/rejected": -406.0,
      "eval_logps/rejected_bottom_tokens": -14.3125,
      "eval_logps/rejected_top_tokens": -0.000759124755859375,
      "eval_loss": 0.6303857564926147,
      "eval_rewards/accuracies": 0.6626865863800049,
      "eval_rewards/chosen": -0.451171875,
      "eval_rewards/margins": 0.2275390625,
      "eval_rewards/rejected": -0.6796875,
      "eval_runtime": 111.5027,
      "eval_samples_per_second": 17.937,
      "eval_steps_per_second": 0.601,
      "step": 300
    },
    {
      "epoch": 0.6084396467124632,
      "grad_norm": 3.1412458629194835,
      "learning_rate": 1.9891702030116897e-07,
      "logits/chosen": -1.140625,
      "logits/rejected": -1.2421875,
      "logps/chosen": -446.0,
      "logps/chosen_bottom_tokens": -14.375,
      "logps/chosen_top_tokens": -0.00074005126953125,
      "logps/rejected": -358.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.0007171630859375,
      "loss": 0.6234,
      "rewards/accuracies": 0.6949999928474426,
      "rewards/chosen": -0.408203125,
      "rewards/margins": 0.2451171875,
      "rewards/rejected": -0.65625,
      "step": 310
    },
    {
      "epoch": 0.6280667320902846,
      "grad_norm": 3.1923082526436986,
      "learning_rate": 1.8226368099107792e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -1.2109375,
      "logps/chosen": -424.0,
      "logps/chosen_bottom_tokens": -14.3125,
      "logps/chosen_top_tokens": -0.00080108642578125,
      "logps/rejected": -364.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.000843048095703125,
      "loss": 0.6241,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.42578125,
      "rewards/margins": 0.2216796875,
      "rewards/rejected": -0.6484375,
      "step": 320
    },
    {
      "epoch": 0.647693817468106,
      "grad_norm": 3.064211696764281,
      "learning_rate": 1.6592892284269594e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.2109375,
      "logps/chosen": -408.0,
      "logps/chosen_bottom_tokens": -14.375,
      "logps/chosen_top_tokens": -0.0007171630859375,
      "logps/rejected": -386.0,
      "logps/rejected_bottom_tokens": -14.3125,
      "logps/rejected_top_tokens": -0.000732421875,
      "loss": 0.6224,
      "rewards/accuracies": 0.6799999475479126,
      "rewards/chosen": -0.431640625,
      "rewards/margins": 0.259765625,
      "rewards/rejected": -0.69140625,
      "step": 330
    },
    {
      "epoch": 0.6673209028459274,
      "grad_norm": 3.1791023826814353,
      "learning_rate": 1.4998957238429172e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -1.2421875,
      "logps/chosen": -408.0,
      "logps/chosen_bottom_tokens": -14.3125,
      "logps/chosen_top_tokens": -0.000804901123046875,
      "logps/rejected": -380.0,
      "logps/rejected_bottom_tokens": -14.3125,
      "logps/rejected_top_tokens": -0.00077056884765625,
      "loss": 0.6204,
      "rewards/accuracies": 0.6300000548362732,
      "rewards/chosen": -0.5078125,
      "rewards/margins": 0.244140625,
      "rewards/rejected": -0.75390625,
      "step": 340
    },
    {
      "epoch": 0.6869479882237488,
      "grad_norm": 3.295570474728778,
      "learning_rate": 1.345205964410517e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -1.21875,
      "logps/chosen": -392.0,
      "logps/chosen_bottom_tokens": -14.375,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -372.0,
      "logps/rejected_bottom_tokens": -14.3125,
      "logps/rejected_top_tokens": -0.00089263916015625,
      "loss": 0.627,
      "rewards/accuracies": 0.5850000381469727,
      "rewards/chosen": -0.49609375,
      "rewards/margins": 0.2236328125,
      "rewards/rejected": -0.71875,
      "step": 350
    },
    {
      "epoch": 0.7065750736015701,
      "grad_norm": 3.5211819482445184,
      "learning_rate": 1.1959474954700665e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.21875,
      "logps/chosen": -424.0,
      "logps/chosen_bottom_tokens": -14.375,
      "logps/chosen_top_tokens": -0.00067138671875,
      "logps/rejected": -416.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.000675201416015625,
      "loss": 0.613,
      "rewards/accuracies": 0.7049999833106995,
      "rewards/chosen": -0.44140625,
      "rewards/margins": 0.234375,
      "rewards/rejected": -0.67578125,
      "step": 360
    },
    {
      "epoch": 0.7262021589793916,
      "grad_norm": 3.3333877037469026,
      "learning_rate": 1.0528223176192615e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -1.234375,
      "logps/chosen": -442.0,
      "logps/chosen_bottom_tokens": -14.375,
      "logps/chosen_top_tokens": -0.00069427490234375,
      "logps/rejected": -398.0,
      "logps/rejected_bottom_tokens": -14.375,
      "logps/rejected_top_tokens": -0.00067901611328125,
      "loss": 0.6218,
      "rewards/accuracies": 0.6250000596046448,
      "rewards/chosen": -0.5234375,
      "rewards/margins": 0.2138671875,
      "rewards/rejected": -0.73828125,
      "step": 370
    },
    {
      "epoch": 0.745829244357213,
      "grad_norm": 3.3039144354882657,
      "learning_rate": 9.16503585025567e-08,
      "logits/chosen": -1.1953125,
      "logits/rejected": -1.21875,
      "logps/chosen": -420.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.00090789794921875,
      "logps/rejected": -412.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.00104522705078125,
      "loss": 0.6279,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5,
      "rewards/margins": 0.2041015625,
      "rewards/rejected": -0.703125,
      "step": 380
    },
    {
      "epoch": 0.7654563297350343,
      "grad_norm": 3.460907844274303,
      "learning_rate": 7.876324394107017e-08,
      "logits/chosen": -1.15625,
      "logits/rejected": -1.203125,
      "logps/chosen": -442.0,
      "logps/chosen_bottom_tokens": -14.3125,
      "logps/chosen_top_tokens": -0.00067901611328125,
      "logps/rejected": -418.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.00069427490234375,
      "loss": 0.6289,
      "rewards/accuracies": 0.6350000500679016,
      "rewards/chosen": -0.50390625,
      "rewards/margins": 0.2255859375,
      "rewards/rejected": -0.7265625,
      "step": 390
    },
    {
      "epoch": 0.7850834151128557,
      "grad_norm": 3.2842912290921897,
      "learning_rate": 6.668149945978201e-08,
      "logits/chosen": -1.1953125,
      "logits/rejected": -1.2265625,
      "logps/chosen": -440.0,
      "logps/chosen_bottom_tokens": -14.3125,
      "logps/chosen_top_tokens": -0.00070953369140625,
      "logps/rejected": -420.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.000732421875,
      "loss": 0.6102,
      "rewards/accuracies": 0.6700000166893005,
      "rewards/chosen": -0.482421875,
      "rewards/margins": 0.291015625,
      "rewards/rejected": -0.7734375,
      "step": 400
    },
    {
      "epoch": 0.7850834151128557,
      "eval_logits/chosen": -1.1875,
      "eval_logits/rejected": -1.234375,
      "eval_logps/chosen": -444.0,
      "eval_logps/chosen_bottom_tokens": -14.3125,
      "eval_logps/chosen_top_tokens": -0.00067138671875,
      "eval_logps/rejected": -414.0,
      "eval_logps/rejected_bottom_tokens": -14.25,
      "eval_logps/rejected_top_tokens": -0.00066375732421875,
      "eval_loss": 0.6267920136451721,
      "eval_rewards/accuracies": 0.6567164063453674,
      "eval_rewards/chosen": -0.50390625,
      "eval_rewards/margins": 0.2578125,
      "eval_rewards/rejected": -0.76171875,
      "eval_runtime": 111.5791,
      "eval_samples_per_second": 17.925,
      "eval_steps_per_second": 0.6,
      "step": 400
    },
    {
      "epoch": 0.8047105004906772,
      "grad_norm": 3.3007954730404303,
      "learning_rate": 5.546194858038072e-08,
      "logits/chosen": -1.171875,
      "logits/rejected": -1.21875,
      "logps/chosen": -416.0,
      "logps/chosen_bottom_tokens": -14.3125,
      "logps/chosen_top_tokens": -0.000766754150390625,
      "logps/rejected": -374.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.000827789306640625,
      "loss": 0.6227,
      "rewards/accuracies": 0.6699999570846558,
      "rewards/chosen": -0.51171875,
      "rewards/margins": 0.28515625,
      "rewards/rejected": -0.796875,
      "step": 410
    },
    {
      "epoch": 0.8243375858684985,
      "grad_norm": 3.9743687860867185,
      "learning_rate": 4.5157359708432626e-08,
      "logits/chosen": -1.1953125,
      "logits/rejected": -1.2265625,
      "logps/chosen": -394.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.00075531005859375,
      "logps/rejected": -412.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.0007476806640625,
      "loss": 0.6205,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.474609375,
      "rewards/margins": 0.234375,
      "rewards/rejected": -0.70703125,
      "step": 420
    },
    {
      "epoch": 0.8439646712463199,
      "grad_norm": 3.1969688623984633,
      "learning_rate": 3.581619795012874e-08,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.1875,
      "logps/chosen": -400.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000782012939453125,
      "logps/rejected": -404.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.000743865966796875,
      "loss": 0.6208,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.474609375,
      "rewards/margins": 0.267578125,
      "rewards/rejected": -0.7421875,
      "step": 430
    },
    {
      "epoch": 0.8635917566241413,
      "grad_norm": 3.705663203159775,
      "learning_rate": 2.748239716854589e-08,
      "logits/chosen": -1.2109375,
      "logits/rejected": -1.1953125,
      "logps/chosen": -424.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000728607177734375,
      "logps/rejected": -420.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.000751495361328125,
      "loss": 0.6398,
      "rewards/accuracies": 0.5849999785423279,
      "rewards/chosen": -0.51171875,
      "rewards/margins": 0.1435546875,
      "rewards/rejected": -0.65625,
      "step": 440
    },
    {
      "epoch": 0.8832188420019627,
      "grad_norm": 3.9792023056235455,
      "learning_rate": 2.0195153351498323e-08,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.2109375,
      "logps/chosen": -432.0,
      "logps/chosen_bottom_tokens": -14.375,
      "logps/chosen_top_tokens": -0.000732421875,
      "logps/rejected": -420.0,
      "logps/rejected_bottom_tokens": -14.3125,
      "logps/rejected_top_tokens": -0.000705718994140625,
      "loss": 0.611,
      "rewards/accuracies": 0.6149999499320984,
      "rewards/chosen": -0.53125,
      "rewards/margins": 0.2421875,
      "rewards/rejected": -0.7734375,
      "step": 450
    },
    {
      "epoch": 0.9028459273797841,
      "grad_norm": 3.598443005581659,
      "learning_rate": 1.3988740262822846e-08,
      "logits/chosen": -1.1953125,
      "logits/rejected": -1.203125,
      "logps/chosen": -428.0,
      "logps/chosen_bottom_tokens": -14.375,
      "logps/chosen_top_tokens": -0.00067138671875,
      "logps/rejected": -410.0,
      "logps/rejected_bottom_tokens": -14.3125,
      "logps/rejected_top_tokens": -0.000667572021484375,
      "loss": 0.6138,
      "rewards/accuracies": 0.6349999904632568,
      "rewards/chosen": -0.490234375,
      "rewards/margins": 0.216796875,
      "rewards/rejected": -0.70703125,
      "step": 460
    },
    {
      "epoch": 0.9224730127576055,
      "grad_norm": 3.423571391469107,
      "learning_rate": 8.892348244137788e-09,
      "logits/chosen": -1.1875,
      "logits/rejected": -1.2421875,
      "logps/chosen": -474.0,
      "logps/chosen_bottom_tokens": -14.5625,
      "logps/chosen_top_tokens": -0.000675201416015625,
      "logps/rejected": -444.0,
      "logps/rejected_bottom_tokens": -14.375,
      "logps/rejected_top_tokens": -0.000652313232421875,
      "loss": 0.6106,
      "rewards/accuracies": 0.6299999952316284,
      "rewards/chosen": -0.494140625,
      "rewards/margins": 0.271484375,
      "rewards/rejected": -0.765625,
      "step": 470
    },
    {
      "epoch": 0.9421000981354269,
      "grad_norm": 3.1667123948106584,
      "learning_rate": 4.929946925231076e-09,
      "logits/chosen": -1.1328125,
      "logits/rejected": -1.171875,
      "logps/chosen": -410.0,
      "logps/chosen_bottom_tokens": -14.375,
      "logps/chosen_top_tokens": -0.0007476806640625,
      "logps/rejected": -412.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.00095367431640625,
      "loss": 0.6203,
      "rewards/accuracies": 0.6049999594688416,
      "rewards/chosen": -0.48046875,
      "rewards/margins": 0.1953125,
      "rewards/rejected": -0.67578125,
      "step": 480
    },
    {
      "epoch": 0.9617271835132483,
      "grad_norm": 3.5902417143779024,
      "learning_rate": 2.1201724887858484e-09,
      "logits/chosen": -1.1640625,
      "logits/rejected": -1.171875,
      "logps/chosen": -422.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.00072479248046875,
      "logps/rejected": -412.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.000743865966796875,
      "loss": 0.6235,
      "rewards/accuracies": 0.5949999690055847,
      "rewards/chosen": -0.5390625,
      "rewards/margins": 0.2265625,
      "rewards/rejected": -0.765625,
      "step": 490
    },
    {
      "epoch": 0.9813542688910697,
      "grad_norm": 3.3154898943344704,
      "learning_rate": 4.762400196664518e-10,
      "logits/chosen": -1.1484375,
      "logits/rejected": -1.1953125,
      "logps/chosen": -428.0,
      "logps/chosen_bottom_tokens": -14.375,
      "logps/chosen_top_tokens": -0.000621795654296875,
      "logps/rejected": -388.0,
      "logps/rejected_bottom_tokens": -14.3125,
      "logps/rejected_top_tokens": -0.0006256103515625,
      "loss": 0.6084,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.578125,
      "rewards/margins": 0.1787109375,
      "rewards/rejected": -0.75390625,
      "step": 500
    },
    {
      "epoch": 0.9813542688910697,
      "eval_logits/chosen": -1.1953125,
      "eval_logits/rejected": -1.2421875,
      "eval_logps/chosen": -446.0,
      "eval_logps/chosen_bottom_tokens": -14.375,
      "eval_logps/chosen_top_tokens": -0.000743865966796875,
      "eval_logps/rejected": -416.0,
      "eval_logps/rejected_bottom_tokens": -14.3125,
      "eval_logps/rejected_top_tokens": -0.0007476806640625,
      "eval_loss": 0.6259472370147705,
      "eval_rewards/accuracies": 0.6567164659500122,
      "eval_rewards/chosen": -0.5234375,
      "eval_rewards/margins": 0.26171875,
      "eval_rewards/rejected": -0.78515625,
      "eval_runtime": 111.4505,
      "eval_samples_per_second": 17.945,
      "eval_steps_per_second": 0.601,
      "step": 500
    },
    {
      "epoch": 1.000981354268891,
      "grad_norm": 3.606361815253299,
      "learning_rate": 4.99999941211936e-07,
      "logits/chosen": -1.1953125,
      "logits/rejected": -1.234375,
      "logps/chosen": -486.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000675201416015625,
      "logps/rejected": -450.0,
      "logps/rejected_bottom_tokens": -14.3125,
      "logps/rejected_top_tokens": -0.000675201416015625,
      "loss": 0.6145,
      "rewards/accuracies": 0.6649999618530273,
      "rewards/chosen": -0.51953125,
      "rewards/margins": 0.2138671875,
      "rewards/rejected": -0.734375,
      "step": 510
    },
    {
      "epoch": 1.0206084396467126,
      "grad_norm": 3.5675558410073815,
      "learning_rate": 4.999928866777183e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -1.203125,
      "logps/chosen": -440.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.00072479248046875,
      "logps/rejected": -386.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000720977783203125,
      "loss": 0.6316,
      "rewards/accuracies": 0.6800000071525574,
      "rewards/chosen": -0.546875,
      "rewards/margins": 0.26953125,
      "rewards/rejected": -0.8203125,
      "step": 520
    },
    {
      "epoch": 1.0402355250245339,
      "grad_norm": 3.7459335473757993,
      "learning_rate": 4.999740749108743e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.2265625,
      "logps/chosen": -466.0,
      "logps/chosen_bottom_tokens": -14.3125,
      "logps/chosen_top_tokens": -0.000682830810546875,
      "logps/rejected": -428.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.000652313232421875,
      "loss": 0.6021,
      "rewards/accuracies": 0.6950000524520874,
      "rewards/chosen": -0.546875,
      "rewards/margins": 0.29296875,
      "rewards/rejected": -0.83984375,
      "step": 530
    },
    {
      "epoch": 1.0598626104023552,
      "grad_norm": 3.722276672156144,
      "learning_rate": 4.999435067961269e-07,
      "logits/chosen": -1.1640625,
      "logits/rejected": -1.203125,
      "logps/chosen": -426.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.00066375732421875,
      "logps/rejected": -418.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.0006866455078125,
      "loss": 0.6311,
      "rewards/accuracies": 0.6299999952316284,
      "rewards/chosen": -0.578125,
      "rewards/margins": 0.2275390625,
      "rewards/rejected": -0.8046875,
      "step": 540
    },
    {
      "epoch": 1.0794896957801767,
      "grad_norm": 3.8251776486836486,
      "learning_rate": 4.999011837711027e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -1.2265625,
      "logps/chosen": -428.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.00058746337890625,
      "logps/rejected": -400.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.000579833984375,
      "loss": 0.6112,
      "rewards/accuracies": 0.6100000143051147,
      "rewards/chosen": -0.57421875,
      "rewards/margins": 0.2353515625,
      "rewards/rejected": -0.80859375,
      "step": 550
    },
    {
      "epoch": 1.099116781157998,
      "grad_norm": 3.7487868364304684,
      "learning_rate": 4.998471078262648e-07,
      "logits/chosen": -1.15625,
      "logits/rejected": -1.171875,
      "logps/chosen": -430.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000652313232421875,
      "logps/rejected": -414.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000652313232421875,
      "loss": 0.6098,
      "rewards/accuracies": 0.6899999976158142,
      "rewards/chosen": -0.546875,
      "rewards/margins": 0.265625,
      "rewards/rejected": -0.8125,
      "step": 560
    },
    {
      "epoch": 1.1187438665358194,
      "grad_norm": 5.204331585494727,
      "learning_rate": 4.997812815048196e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -1.2734375,
      "logps/chosen": -408.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.00057220458984375,
      "logps/rejected": -412.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.0005645751953125,
      "loss": 0.6146,
      "rewards/accuracies": 0.6899999976158142,
      "rewards/chosen": -0.515625,
      "rewards/margins": 0.341796875,
      "rewards/rejected": -0.859375,
      "step": 570
    },
    {
      "epoch": 1.138370951913641,
      "grad_norm": 3.6081766106504736,
      "learning_rate": 4.997037079025965e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -1.2265625,
      "logps/chosen": -444.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.00054168701171875,
      "logps/rejected": -426.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.000553131103515625,
      "loss": 0.6048,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.59375,
      "rewards/margins": 0.2734375,
      "rewards/rejected": -0.8671875,
      "step": 580
    },
    {
      "epoch": 1.1579980372914622,
      "grad_norm": 4.870398963575735,
      "learning_rate": 4.996143906679027e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -1.2421875,
      "logps/chosen": -432.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000499725341796875,
      "logps/rejected": -408.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.00052642822265625,
      "loss": 0.6143,
      "rewards/accuracies": 0.6299999952316284,
      "rewards/chosen": -0.57421875,
      "rewards/margins": 0.22265625,
      "rewards/rejected": -0.796875,
      "step": 590
    },
    {
      "epoch": 1.1776251226692835,
      "grad_norm": 3.826862126663985,
      "learning_rate": 4.995133340013521e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -1.25,
      "logps/chosen": -442.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000606536865234375,
      "logps/rejected": -424.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000545501708984375,
      "loss": 0.6115,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.58203125,
      "rewards/margins": 0.267578125,
      "rewards/rejected": -0.8515625,
      "step": 600
    },
    {
      "epoch": 1.1776251226692835,
      "eval_logits/chosen": -1.2109375,
      "eval_logits/rejected": -1.2578125,
      "eval_logps/chosen": -450.0,
      "eval_logps/chosen_bottom_tokens": -14.25,
      "eval_logps/chosen_top_tokens": -0.00060272216796875,
      "eval_logps/rejected": -426.0,
      "eval_logps/rejected_bottom_tokens": -14.125,
      "eval_logps/rejected_top_tokens": -0.00058746337890625,
      "eval_loss": 0.6120752096176147,
      "eval_rewards/accuracies": 0.6805970072746277,
      "eval_rewards/chosen": -0.5546875,
      "eval_rewards/margins": 0.32421875,
      "eval_rewards/rejected": -0.87890625,
      "eval_runtime": 107.8869,
      "eval_samples_per_second": 18.538,
      "eval_steps_per_second": 0.621,
      "step": 600
    },
    {
      "epoch": 1.197252208047105,
      "grad_norm": 3.870372977700712,
      "learning_rate": 4.994005426556668e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -1.21875,
      "logps/chosen": -406.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000598907470703125,
      "logps/rejected": -412.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.000621795654296875,
      "loss": 0.6124,
      "rewards/accuracies": 0.6550000309944153,
      "rewards/chosen": -0.5390625,
      "rewards/margins": 0.28125,
      "rewards/rejected": -0.8203125,
      "step": 610
    },
    {
      "epoch": 1.2168792934249264,
      "grad_norm": 4.186566429764424,
      "learning_rate": 4.99276021935454e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -1.265625,
      "logps/chosen": -430.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.00051116943359375,
      "logps/rejected": -400.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.0005035400390625,
      "loss": 0.6022,
      "rewards/accuracies": 0.7150000333786011,
      "rewards/chosen": -0.40625,
      "rewards/margins": 0.34375,
      "rewards/rejected": -0.75,
      "step": 620
    },
    {
      "epoch": 1.2365063788027477,
      "grad_norm": 3.609372953026834,
      "learning_rate": 4.991397776969566e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -1.25,
      "logps/chosen": -448.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000659942626953125,
      "logps/rejected": -398.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000629425048828125,
      "loss": 0.6018,
      "rewards/accuracies": 0.6950000524520874,
      "rewards/chosen": -0.6015625,
      "rewards/margins": 0.30078125,
      "rewards/rejected": -0.90234375,
      "step": 630
    },
    {
      "epoch": 1.2561334641805693,
      "grad_norm": 4.171667413733576,
      "learning_rate": 4.989918163477777e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -1.2734375,
      "logps/chosen": -424.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.00052642822265625,
      "logps/rejected": -394.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.00052642822265625,
      "loss": 0.5944,
      "rewards/accuracies": 0.6700000166893005,
      "rewards/chosen": -0.58203125,
      "rewards/margins": 0.333984375,
      "rewards/rejected": -0.9140625,
      "step": 640
    },
    {
      "epoch": 1.2757605495583906,
      "grad_norm": 4.098077395806722,
      "learning_rate": 4.988321448465795e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.28125,
      "logps/chosen": -394.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.0005645751953125,
      "logps/rejected": -414.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.000576019287109375,
      "loss": 0.6045,
      "rewards/accuracies": 0.6650000810623169,
      "rewards/chosen": -0.57421875,
      "rewards/margins": 0.322265625,
      "rewards/rejected": -0.8984375,
      "step": 650
    },
    {
      "epoch": 1.295387634936212,
      "grad_norm": 3.766274950371864,
      "learning_rate": 4.986607707027556e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -1.2421875,
      "logps/chosen": -406.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000614166259765625,
      "logps/rejected": -402.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.0006103515625,
      "loss": 0.5906,
      "rewards/accuracies": 0.6949999928474426,
      "rewards/chosen": -0.625,
      "rewards/margins": 0.3046875,
      "rewards/rejected": -0.9296875,
      "step": 660
    },
    {
      "epoch": 1.3150147203140334,
      "grad_norm": 4.3244931656081755,
      "learning_rate": 4.984777019760778e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -1.2265625,
      "logps/chosen": -444.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00083160400390625,
      "logps/rejected": -428.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000812530517578125,
      "loss": 0.5862,
      "rewards/accuracies": 0.6399999856948853,
      "rewards/chosen": -0.63671875,
      "rewards/margins": 0.322265625,
      "rewards/rejected": -0.9609375,
      "step": 670
    },
    {
      "epoch": 1.3346418056918548,
      "grad_norm": 5.114477587957832,
      "learning_rate": 4.982829472763176e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -1.2734375,
      "logps/chosen": -446.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000568389892578125,
      "logps/rejected": -440.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.000553131103515625,
      "loss": 0.6053,
      "rewards/accuracies": 0.6550000309944153,
      "rewards/chosen": -0.59375,
      "rewards/margins": 0.376953125,
      "rewards/rejected": -0.97265625,
      "step": 680
    },
    {
      "epoch": 1.354268891069676,
      "grad_norm": 4.909788993759657,
      "learning_rate": 4.98076515762841e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -1.2734375,
      "logps/chosen": -452.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000682830810546875,
      "logps/rejected": -430.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00063323974609375,
      "loss": 0.604,
      "rewards/accuracies": 0.6299999952316284,
      "rewards/chosen": -0.78125,
      "rewards/margins": 0.28125,
      "rewards/rejected": -1.0625,
      "step": 690
    },
    {
      "epoch": 1.3738959764474976,
      "grad_norm": 4.433097765275912,
      "learning_rate": 4.978584171441774e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -1.2734375,
      "logps/chosen": -448.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.00063323974609375,
      "logps/rejected": -432.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.0006256103515625,
      "loss": 0.607,
      "rewards/accuracies": 0.6399999856948853,
      "rewards/chosen": -0.81640625,
      "rewards/margins": 0.298828125,
      "rewards/rejected": -1.1171875,
      "step": 700
    },
    {
      "epoch": 1.3738959764474976,
      "eval_logits/chosen": -1.234375,
      "eval_logits/rejected": -1.28125,
      "eval_logps/chosen": -460.0,
      "eval_logps/chosen_bottom_tokens": -14.1875,
      "eval_logps/chosen_top_tokens": -0.0006103515625,
      "eval_logps/rejected": -438.0,
      "eval_logps/rejected_bottom_tokens": -14.125,
      "eval_logps/rejected_top_tokens": -0.000579833984375,
      "eval_loss": 0.6068310737609863,
      "eval_rewards/accuracies": 0.6985074877738953,
      "eval_rewards/chosen": -0.6640625,
      "eval_rewards/margins": 0.341796875,
      "eval_rewards/rejected": -1.0078125,
      "eval_runtime": 107.7706,
      "eval_samples_per_second": 18.558,
      "eval_steps_per_second": 0.622,
      "step": 700
    },
    {
      "epoch": 1.393523061825319,
      "grad_norm": 3.765327199305878,
      "learning_rate": 4.976286616775634e-07,
      "logits/chosen": -1.234375,
      "logits/rejected": -1.265625,
      "logps/chosen": -418.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000553131103515625,
      "logps/rejected": -396.0,
      "logps/rejected_bottom_tokens": -14.3125,
      "logps/rejected_top_tokens": -0.00054168701171875,
      "loss": 0.6011,
      "rewards/accuracies": 0.6450001001358032,
      "rewards/chosen": -0.61328125,
      "rewards/margins": 0.279296875,
      "rewards/rejected": -0.890625,
      "step": 710
    },
    {
      "epoch": 1.4131501472031402,
      "grad_norm": 3.805472086883035,
      "learning_rate": 4.973872601684603e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -1.28125,
      "logps/chosen": -480.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.00057220458984375,
      "logps/rejected": -448.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.0005340576171875,
      "loss": 0.5884,
      "rewards/accuracies": 0.7050000429153442,
      "rewards/chosen": -0.52734375,
      "rewards/margins": 0.3046875,
      "rewards/rejected": -0.83203125,
      "step": 720
    },
    {
      "epoch": 1.4327772325809618,
      "grad_norm": 4.230218832508928,
      "learning_rate": 4.971342239700461e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -1.25,
      "logps/chosen": -442.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000644683837890625,
      "logps/rejected": -410.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000640869140625,
      "loss": 0.5926,
      "rewards/accuracies": 0.7100000381469727,
      "rewards/chosen": -0.6328125,
      "rewards/margins": 0.3828125,
      "rewards/rejected": -1.015625,
      "step": 730
    },
    {
      "epoch": 1.452404317958783,
      "grad_norm": 4.491255520471624,
      "learning_rate": 4.96869564982681e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.28125,
      "logps/chosen": -414.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.00054931640625,
      "logps/rejected": -434.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000522613525390625,
      "loss": 0.6027,
      "rewards/accuracies": 0.6750000715255737,
      "rewards/chosen": -0.71484375,
      "rewards/margins": 0.404296875,
      "rewards/rejected": -1.1171875,
      "step": 740
    },
    {
      "epoch": 1.4720314033366044,
      "grad_norm": 4.442897832667683,
      "learning_rate": 4.965932956533486e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -1.28125,
      "logps/chosen": -464.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000591278076171875,
      "logps/rejected": -450.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00055694580078125,
      "loss": 0.5909,
      "rewards/accuracies": 0.6950000524520874,
      "rewards/chosen": -0.7734375,
      "rewards/margins": 0.40625,
      "rewards/rejected": -1.1796875,
      "step": 750
    },
    {
      "epoch": 1.491658488714426,
      "grad_norm": 4.749468874704089,
      "learning_rate": 4.963054289750692e-07,
      "logits/chosen": -1.1875,
      "logits/rejected": -1.2265625,
      "logps/chosen": -454.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000568389892578125,
      "logps/rejected": -454.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00052642822265625,
      "loss": 0.5867,
      "rewards/accuracies": 0.6949999928474426,
      "rewards/chosen": -0.79296875,
      "rewards/margins": 0.359375,
      "rewards/rejected": -1.15625,
      "step": 760
    },
    {
      "epoch": 1.5112855740922473,
      "grad_norm": 5.388562265861462,
      "learning_rate": 4.960059784862905e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -1.28125,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000553131103515625,
      "logps/rejected": -446.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00054931640625,
      "loss": 0.5894,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.71875,
      "rewards/margins": 0.32421875,
      "rewards/rejected": -1.046875,
      "step": 770
    },
    {
      "epoch": 1.5309126594700686,
      "grad_norm": 5.21775194242665,
      "learning_rate": 4.956949582702491e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -1.296875,
      "logps/chosen": -448.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00067138671875,
      "logps/rejected": -418.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000621795654296875,
      "loss": 0.5858,
      "rewards/accuracies": 0.7399999499320984,
      "rewards/chosen": -0.69140625,
      "rewards/margins": 0.419921875,
      "rewards/rejected": -1.109375,
      "step": 780
    },
    {
      "epoch": 1.5505397448478901,
      "grad_norm": 4.798321151943007,
      "learning_rate": 4.953723829543095e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -1.28125,
      "logps/chosen": -460.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.00060272216796875,
      "logps/rejected": -458.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.000591278076171875,
      "loss": 0.6019,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.75,
      "rewards/margins": 0.29296875,
      "rewards/rejected": -1.0390625,
      "step": 790
    },
    {
      "epoch": 1.5701668302257115,
      "grad_norm": 4.565201285919172,
      "learning_rate": 4.950382677092754e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -1.296875,
      "logps/chosen": -470.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000637054443359375,
      "logps/rejected": -468.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.000576019287109375,
      "loss": 0.5764,
      "rewards/accuracies": 0.6399999856948853,
      "rewards/chosen": -0.76171875,
      "rewards/margins": 0.310546875,
      "rewards/rejected": -1.0703125,
      "step": 800
    },
    {
      "epoch": 1.5701668302257115,
      "eval_logits/chosen": -1.265625,
      "eval_logits/rejected": -1.3125,
      "eval_logps/chosen": -468.0,
      "eval_logps/chosen_bottom_tokens": -14.25,
      "eval_logps/chosen_top_tokens": -0.00067901611328125,
      "eval_logps/rejected": -452.0,
      "eval_logps/rejected_bottom_tokens": -14.125,
      "eval_logps/rejected_top_tokens": -0.00067138671875,
      "eval_loss": 0.5995947122573853,
      "eval_rewards/accuracies": 0.6865671873092651,
      "eval_rewards/chosen": -0.75,
      "eval_rewards/margins": 0.388671875,
      "eval_rewards/rejected": -1.140625,
      "eval_runtime": 107.7895,
      "eval_samples_per_second": 18.555,
      "eval_steps_per_second": 0.622,
      "step": 800
    },
    {
      "epoch": 1.5897939156035328,
      "grad_norm": 4.401209533392183,
      "learning_rate": 4.946926282486765e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -1.28125,
      "logps/chosen": -468.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.00064849853515625,
      "logps/rejected": -458.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.00061798095703125,
      "loss": 0.5877,
      "rewards/accuracies": 0.6799999475479126,
      "rewards/chosen": -0.75,
      "rewards/margins": 0.39453125,
      "rewards/rejected": -1.140625,
      "step": 810
    },
    {
      "epoch": 1.6094210009813543,
      "grad_norm": 4.172999078191103,
      "learning_rate": 4.943354808280297e-07,
      "logits/chosen": -1.2265625,
      "logits/rejected": -1.2421875,
      "logps/chosen": -424.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000598907470703125,
      "logps/rejected": -428.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.000637054443359375,
      "loss": 0.6005,
      "rewards/accuracies": 0.6600000262260437,
      "rewards/chosen": -0.671875,
      "rewards/margins": 0.369140625,
      "rewards/rejected": -1.0390625,
      "step": 820
    },
    {
      "epoch": 1.6290480863591756,
      "grad_norm": 4.682327663420403,
      "learning_rate": 4.93966842244074e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.2890625,
      "logps/chosen": -458.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.00058746337890625,
      "logps/rejected": -416.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00054168701171875,
      "loss": 0.5873,
      "rewards/accuracies": 0.7150000333786011,
      "rewards/chosen": -0.65625,
      "rewards/margins": 0.3515625,
      "rewards/rejected": -1.0078125,
      "step": 830
    },
    {
      "epoch": 1.648675171736997,
      "grad_norm": 3.971337026667362,
      "learning_rate": 4.93586729833981e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -1.3125,
      "logps/chosen": -458.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000530242919921875,
      "logps/rejected": -426.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000518798828125,
      "loss": 0.5706,
      "rewards/accuracies": 0.7149999737739563,
      "rewards/chosen": -0.65234375,
      "rewards/margins": 0.443359375,
      "rewards/rejected": -1.09375,
      "step": 840
    },
    {
      "epoch": 1.6683022571148185,
      "grad_norm": 4.9613043407153885,
      "learning_rate": 4.931951614745395e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.296875,
      "logps/chosen": -474.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000499725341796875,
      "logps/rejected": -462.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.000492095947265625,
      "loss": 0.5698,
      "rewards/accuracies": 0.6800000071525574,
      "rewards/chosen": -0.640625,
      "rewards/margins": 0.4140625,
      "rewards/rejected": -1.0546875,
      "step": 850
    },
    {
      "epoch": 1.6879293424926398,
      "grad_norm": 4.37364089862016,
      "learning_rate": 4.927921555813147e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -1.3203125,
      "logps/chosen": -452.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.0005950927734375,
      "logps/rejected": -430.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.000568389892578125,
      "loss": 0.5829,
      "rewards/accuracies": 0.7049999237060547,
      "rewards/chosen": -0.6015625,
      "rewards/margins": 0.369140625,
      "rewards/rejected": -0.96875,
      "step": 860
    },
    {
      "epoch": 1.7075564278704611,
      "grad_norm": 4.934313256041272,
      "learning_rate": 4.923777311077819e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.296875,
      "logps/chosen": -450.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000640869140625,
      "logps/rejected": -448.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.0005950927734375,
      "loss": 0.5745,
      "rewards/accuracies": 0.6750000715255737,
      "rewards/chosen": -0.66796875,
      "rewards/margins": 0.41015625,
      "rewards/rejected": -1.078125,
      "step": 870
    },
    {
      "epoch": 1.7271835132482827,
      "grad_norm": 4.398593383317482,
      "learning_rate": 4.919519075444358e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.328125,
      "logps/chosen": -458.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000644683837890625,
      "logps/rejected": -432.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.00061798095703125,
      "loss": 0.5903,
      "rewards/accuracies": 0.7149999737739563,
      "rewards/chosen": -0.9296875,
      "rewards/margins": 0.427734375,
      "rewards/rejected": -1.359375,
      "step": 880
    },
    {
      "epoch": 1.746810598626104,
      "grad_norm": 4.552566240750762,
      "learning_rate": 4.915147049178725e-07,
      "logits/chosen": -1.2421875,
      "logits/rejected": -1.3046875,
      "logps/chosen": -468.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000583648681640625,
      "logps/rejected": -448.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.000583648681640625,
      "loss": 0.5833,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -0.6875,
      "rewards/margins": 0.46484375,
      "rewards/rejected": -1.15625,
      "step": 890
    },
    {
      "epoch": 1.7664376840039253,
      "grad_norm": 3.980827367632633,
      "learning_rate": 4.910661437898493e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.3046875,
      "logps/chosen": -464.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.0005340576171875,
      "logps/rejected": -472.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.00055694580078125,
      "loss": 0.5903,
      "rewards/accuracies": 0.6600000262260437,
      "rewards/chosen": -0.67578125,
      "rewards/margins": 0.375,
      "rewards/rejected": -1.046875,
      "step": 900
    },
    {
      "epoch": 1.7664376840039253,
      "eval_logits/chosen": -1.265625,
      "eval_logits/rejected": -1.3125,
      "eval_logps/chosen": -452.0,
      "eval_logps/chosen_bottom_tokens": -14.25,
      "eval_logps/chosen_top_tokens": -0.0005950927734375,
      "eval_logps/rejected": -434.0,
      "eval_logps/rejected_bottom_tokens": -14.125,
      "eval_logps/rejected_top_tokens": -0.00057220458984375,
      "eval_loss": 0.5984472632408142,
      "eval_rewards/accuracies": 0.7044776082038879,
      "eval_rewards/chosen": -0.58984375,
      "eval_rewards/margins": 0.376953125,
      "eval_rewards/rejected": -0.96484375,
      "eval_runtime": 107.7395,
      "eval_samples_per_second": 18.563,
      "eval_steps_per_second": 0.622,
      "step": 900
    },
    {
      "epoch": 1.7860647693817469,
      "grad_norm": 4.250190572849706,
      "learning_rate": 4.906062452563164e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.28125,
      "logps/chosen": -424.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.00064849853515625,
      "logps/rejected": -430.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000614166259765625,
      "loss": 0.5943,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.67578125,
      "rewards/margins": 0.26171875,
      "rewards/rejected": -0.9375,
      "step": 910
    },
    {
      "epoch": 1.8056918547595682,
      "grad_norm": 4.240685239707214,
      "learning_rate": 4.901350309464256e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -1.296875,
      "logps/chosen": -458.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000522613525390625,
      "logps/rejected": -444.0,
      "logps/rejected_bottom_tokens": -14.25,
      "logps/rejected_top_tokens": -0.00052642822265625,
      "loss": 0.5827,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -0.64453125,
      "rewards/margins": 0.380859375,
      "rewards/rejected": -1.0234375,
      "step": 920
    },
    {
      "epoch": 1.8253189401373895,
      "grad_norm": 4.024817983900379,
      "learning_rate": 4.896525230215123e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -1.3046875,
      "logps/chosen": -450.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000507354736328125,
      "logps/rejected": -450.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000530242919921875,
      "loss": 0.5686,
      "rewards/accuracies": 0.7150000333786011,
      "rewards/chosen": -0.640625,
      "rewards/margins": 0.490234375,
      "rewards/rejected": -1.1328125,
      "step": 930
    },
    {
      "epoch": 1.844946025515211,
      "grad_norm": 4.38980118561382,
      "learning_rate": 4.891587441740539e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.328125,
      "logps/chosen": -478.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000518798828125,
      "logps/rejected": -460.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.0005035400390625,
      "loss": 0.5839,
      "rewards/accuracies": 0.6450000405311584,
      "rewards/chosen": -0.76953125,
      "rewards/margins": 0.353515625,
      "rewards/rejected": -1.125,
      "step": 940
    },
    {
      "epoch": 1.8645731108930323,
      "grad_norm": 4.759599103438232,
      "learning_rate": 4.886537176266024e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.3203125,
      "logps/chosen": -438.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000629425048828125,
      "logps/rejected": -414.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000621795654296875,
      "loss": 0.5779,
      "rewards/accuracies": 0.7150000333786011,
      "rewards/chosen": -0.78515625,
      "rewards/margins": 0.373046875,
      "rewards/rejected": -1.15625,
      "step": 950
    },
    {
      "epoch": 1.8842001962708537,
      "grad_norm": 4.786256988626395,
      "learning_rate": 4.881374671306917e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.3125,
      "logps/chosen": -486.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000576019287109375,
      "logps/rejected": -480.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00054931640625,
      "loss": 0.5842,
      "rewards/accuracies": 0.6799999475479126,
      "rewards/chosen": -0.79296875,
      "rewards/margins": 0.4375,
      "rewards/rejected": -1.234375,
      "step": 960
    },
    {
      "epoch": 1.9038272816486752,
      "grad_norm": 4.9859381733928965,
      "learning_rate": 4.876100169657217e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -1.3125,
      "logps/chosen": -444.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000591278076171875,
      "logps/rejected": -430.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000583648681640625,
      "loss": 0.5824,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.76953125,
      "rewards/margins": 0.455078125,
      "rewards/rejected": -1.2265625,
      "step": 970
    },
    {
      "epoch": 1.9234543670264965,
      "grad_norm": 5.322106257872434,
      "learning_rate": 4.87071391937815e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.3046875,
      "logps/chosen": -420.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00075531005859375,
      "logps/rejected": -418.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000732421875,
      "loss": 0.5726,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.81640625,
      "rewards/margins": 0.43359375,
      "rewards/rejected": -1.25,
      "step": 980
    },
    {
      "epoch": 1.9430814524043178,
      "grad_norm": 5.610031785982417,
      "learning_rate": 4.865216173786516e-07,
      "logits/chosen": -1.21875,
      "logits/rejected": -1.2734375,
      "logps/chosen": -464.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000698089599609375,
      "logps/rejected": -444.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.00067901611328125,
      "loss": 0.5806,
      "rewards/accuracies": 0.6800000071525574,
      "rewards/chosen": -0.9296875,
      "rewards/margins": 0.33984375,
      "rewards/rejected": -1.2734375,
      "step": 990
    },
    {
      "epoch": 1.9627085377821394,
      "grad_norm": 4.756477386020176,
      "learning_rate": 4.859607191442768e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.3203125,
      "logps/chosen": -466.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.0006561279296875,
      "logps/rejected": -444.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000667572021484375,
      "loss": 0.5697,
      "rewards/accuracies": 0.6850000023841858,
      "rewards/chosen": -0.796875,
      "rewards/margins": 0.42578125,
      "rewards/rejected": -1.2265625,
      "step": 1000
    },
    {
      "epoch": 1.9627085377821394,
      "eval_logits/chosen": -1.2734375,
      "eval_logits/rejected": -1.3125,
      "eval_logps/chosen": -468.0,
      "eval_logps/chosen_bottom_tokens": -14.0625,
      "eval_logps/chosen_top_tokens": -0.000667572021484375,
      "eval_logps/rejected": -454.0,
      "eval_logps/rejected_bottom_tokens": -14.0,
      "eval_logps/rejected_top_tokens": -0.00061798095703125,
      "eval_loss": 0.5921972393989563,
      "eval_rewards/accuracies": 0.6865671873092651,
      "eval_rewards/chosen": -0.73828125,
      "eval_rewards/margins": 0.416015625,
      "eval_rewards/rejected": -1.15625,
      "eval_runtime": 107.7883,
      "eval_samples_per_second": 18.555,
      "eval_steps_per_second": 0.622,
      "step": 1000
    },
    {
      "epoch": 1.9823356231599607,
      "grad_norm": 4.9024153356053946,
      "learning_rate": 4.85388723613885e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -1.3125,
      "logps/chosen": -482.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000644683837890625,
      "logps/rejected": -458.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000598907470703125,
      "loss": 0.5777,
      "rewards/accuracies": 0.7100000381469727,
      "rewards/chosen": -0.8046875,
      "rewards/margins": 0.423828125,
      "rewards/rejected": -1.2265625,
      "step": 1010
    },
    {
      "epoch": 2.001962708537782,
      "grad_norm": 4.429395780890561,
      "learning_rate": 4.848056576885799e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.3125,
      "logps/chosen": -492.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000560760498046875,
      "logps/rejected": -498.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.0005340576171875,
      "loss": 0.5638,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.828125,
      "rewards/margins": 0.48046875,
      "rewards/rejected": -1.3046875,
      "step": 1020
    },
    {
      "epoch": 2.0215897939156036,
      "grad_norm": 4.728557301310735,
      "learning_rate": 4.842115487901085e-07,
      "logits/chosen": -1.25,
      "logits/rejected": -1.2890625,
      "logps/chosen": -456.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000736236572265625,
      "logps/rejected": -462.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000690460205078125,
      "loss": 0.5488,
      "rewards/accuracies": 0.7199999690055847,
      "rewards/chosen": -0.84375,
      "rewards/margins": 0.46875,
      "rewards/rejected": -1.3125,
      "step": 1030
    },
    {
      "epoch": 2.041216879293425,
      "grad_norm": 4.919488885103522,
      "learning_rate": 4.836064248595719e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.3125,
      "logps/chosen": -456.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.00061798095703125,
      "logps/rejected": -454.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000598907470703125,
      "loss": 0.5544,
      "rewards/accuracies": 0.7199999690055847,
      "rewards/chosen": -0.68359375,
      "rewards/margins": 0.421875,
      "rewards/rejected": -1.109375,
      "step": 1040
    },
    {
      "epoch": 2.060843964671246,
      "grad_norm": 4.398420507697798,
      "learning_rate": 4.829903143561113e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.28125,
      "logps/chosen": -476.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000701904296875,
      "logps/rejected": -484.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000701904296875,
      "loss": 0.5509,
      "rewards/accuracies": 0.7400000691413879,
      "rewards/chosen": -0.74609375,
      "rewards/margins": 0.546875,
      "rewards/rejected": -1.2890625,
      "step": 1050
    },
    {
      "epoch": 2.0804710500490677,
      "grad_norm": 5.763513449763713,
      "learning_rate": 4.82363246255569e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.296875,
      "logps/chosen": -466.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000644683837890625,
      "logps/rejected": -474.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000690460205078125,
      "loss": 0.5693,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -0.90234375,
      "rewards/margins": 0.453125,
      "rewards/rejected": -1.3515625,
      "step": 1060
    },
    {
      "epoch": 2.1000981354268893,
      "grad_norm": 4.588276955004494,
      "learning_rate": 4.817252500491263e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.359375,
      "logps/chosen": -438.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000576019287109375,
      "logps/rejected": -428.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00061798095703125,
      "loss": 0.5475,
      "rewards/accuracies": 0.7499999403953552,
      "rewards/chosen": -0.703125,
      "rewards/margins": 0.52734375,
      "rewards/rejected": -1.2265625,
      "step": 1070
    },
    {
      "epoch": 2.1197252208047104,
      "grad_norm": 4.954842485688175,
      "learning_rate": 4.810763557419163e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.2890625,
      "logps/chosen": -504.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000583648681640625,
      "logps/rejected": -492.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.000576019287109375,
      "loss": 0.565,
      "rewards/accuracies": 0.6699999570846558,
      "rewards/chosen": -0.828125,
      "rewards/margins": 0.4453125,
      "rewards/rejected": -1.2734375,
      "step": 1080
    },
    {
      "epoch": 2.139352306182532,
      "grad_norm": 4.719628678027872,
      "learning_rate": 4.804165938516125e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.328125,
      "logps/chosen": -430.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000629425048828125,
      "logps/rejected": -436.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000606536865234375,
      "loss": 0.5601,
      "rewards/accuracies": 0.6599999666213989,
      "rewards/chosen": -0.83203125,
      "rewards/margins": 0.361328125,
      "rewards/rejected": -1.1953125,
      "step": 1090
    },
    {
      "epoch": 2.1589793915603535,
      "grad_norm": 5.455818727699527,
      "learning_rate": 4.797459954069938e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.3046875,
      "logps/chosen": -436.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00066375732421875,
      "logps/rejected": -442.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000644683837890625,
      "loss": 0.5573,
      "rewards/accuracies": 0.7099999785423279,
      "rewards/chosen": -0.7734375,
      "rewards/margins": 0.412109375,
      "rewards/rejected": -1.1875,
      "step": 1100
    },
    {
      "epoch": 2.1589793915603535,
      "eval_logits/chosen": -1.2890625,
      "eval_logits/rejected": -1.328125,
      "eval_logps/chosen": -476.0,
      "eval_logps/chosen_bottom_tokens": -14.125,
      "eval_logps/chosen_top_tokens": -0.000598907470703125,
      "eval_logps/rejected": -466.0,
      "eval_logps/rejected_bottom_tokens": -14.0,
      "eval_logps/rejected_top_tokens": -0.000560760498046875,
      "eval_loss": 0.5853906273841858,
      "eval_rewards/accuracies": 0.6985074877738953,
      "eval_rewards/chosen": -0.8203125,
      "eval_rewards/margins": 0.45703125,
      "eval_rewards/rejected": -1.28125,
      "eval_runtime": 107.7163,
      "eval_samples_per_second": 18.567,
      "eval_steps_per_second": 0.622,
      "step": 1100
    },
    {
      "epoch": 2.1786064769381746,
      "grad_norm": 4.965235050814131,
      "learning_rate": 4.790645919464854e-07,
      "logits/chosen": -1.265625,
      "logits/rejected": -1.3203125,
      "logps/chosen": -458.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000621795654296875,
      "logps/rejected": -430.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.0005950927734375,
      "loss": 0.5628,
      "rewards/accuracies": 0.7350000143051147,
      "rewards/chosen": -0.87890625,
      "rewards/margins": 0.435546875,
      "rewards/rejected": -1.3125,
      "step": 1110
    },
    {
      "epoch": 2.198233562315996,
      "grad_norm": 5.352102174842767,
      "learning_rate": 4.783724155166751e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.3203125,
      "logps/chosen": -438.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000652313232421875,
      "logps/rejected": -436.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.0006103515625,
      "loss": 0.5384,
      "rewards/accuracies": 0.6850000619888306,
      "rewards/chosen": -0.86328125,
      "rewards/margins": 0.4296875,
      "rewards/rejected": -1.296875,
      "step": 1120
    },
    {
      "epoch": 2.2178606476938176,
      "grad_norm": 5.559211111088488,
      "learning_rate": 4.776694986708061e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.3046875,
      "logps/chosen": -468.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000759124755859375,
      "logps/rejected": -450.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000751495361328125,
      "loss": 0.5466,
      "rewards/accuracies": 0.7249999642372131,
      "rewards/chosen": -0.890625,
      "rewards/margins": 0.458984375,
      "rewards/rejected": -1.3515625,
      "step": 1130
    },
    {
      "epoch": 2.2374877330716387,
      "grad_norm": 5.275098036354786,
      "learning_rate": 4.76955874467247e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -1.3515625,
      "logps/chosen": -468.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.0007171630859375,
      "logps/rejected": -490.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0006866455078125,
      "loss": 0.5552,
      "rewards/accuracies": 0.6949999928474426,
      "rewards/chosen": -0.78515625,
      "rewards/margins": 0.5,
      "rewards/rejected": -1.2890625,
      "step": 1140
    },
    {
      "epoch": 2.2571148184494603,
      "grad_norm": 4.846395975422152,
      "learning_rate": 4.762315764679353e-07,
      "logits/chosen": -1.2890625,
      "logits/rejected": -1.3515625,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00064849853515625,
      "logps/rejected": -472.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.0006561279296875,
      "loss": 0.5436,
      "rewards/accuracies": 0.7350000143051147,
      "rewards/chosen": -0.71875,
      "rewards/margins": 0.5234375,
      "rewards/rejected": -1.2421875,
      "step": 1150
    },
    {
      "epoch": 2.276741903827282,
      "grad_norm": 5.985956261716143,
      "learning_rate": 4.7549663873680074e-07,
      "logits/chosen": -1.2734375,
      "logits/rejected": -1.2890625,
      "logps/chosen": -462.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000762939453125,
      "logps/rejected": -476.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000804901123046875,
      "loss": 0.551,
      "rewards/accuracies": 0.7150000333786011,
      "rewards/chosen": -0.8046875,
      "rewards/margins": 0.458984375,
      "rewards/rejected": -1.265625,
      "step": 1160
    },
    {
      "epoch": 2.296368989205103,
      "grad_norm": 6.348450013593529,
      "learning_rate": 4.7475109583816233e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.375,
      "logps/chosen": -466.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000690460205078125,
      "logps/rejected": -468.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000732421875,
      "loss": 0.55,
      "rewards/accuracies": 0.7300000190734863,
      "rewards/chosen": -0.9609375,
      "rewards/margins": 0.55859375,
      "rewards/rejected": -1.5234375,
      "step": 1170
    },
    {
      "epoch": 2.3159960745829244,
      "grad_norm": 4.561023721100259,
      "learning_rate": 4.739949828351028e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.328125,
      "logps/chosen": -486.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000682830810546875,
      "logps/rejected": -476.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000667572021484375,
      "loss": 0.5583,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9453125,
      "rewards/margins": 0.494140625,
      "rewards/rejected": -1.4375,
      "step": 1180
    },
    {
      "epoch": 2.335623159960746,
      "grad_norm": 5.506208079321664,
      "learning_rate": 4.732283352878199e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.3203125,
      "logps/chosen": -474.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000537872314453125,
      "logps/rejected": -532.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00057220458984375,
      "loss": 0.5368,
      "rewards/accuracies": 0.7350000143051147,
      "rewards/chosen": -0.8984375,
      "rewards/margins": 0.62109375,
      "rewards/rejected": -1.5234375,
      "step": 1190
    },
    {
      "epoch": 2.355250245338567,
      "grad_norm": 5.428963571935742,
      "learning_rate": 4.724511892519537e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.3203125,
      "logps/chosen": -490.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00067901611328125,
      "logps/rejected": -484.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000621795654296875,
      "loss": 0.5439,
      "rewards/accuracies": 0.7350000143051147,
      "rewards/chosen": -1.0390625,
      "rewards/margins": 0.51171875,
      "rewards/rejected": -1.5546875,
      "step": 1200
    },
    {
      "epoch": 2.355250245338567,
      "eval_logits/chosen": -1.328125,
      "eval_logits/rejected": -1.3671875,
      "eval_logps/chosen": -504.0,
      "eval_logps/chosen_bottom_tokens": -14.0625,
      "eval_logps/chosen_top_tokens": -0.000652313232421875,
      "eval_logps/rejected": -498.0,
      "eval_logps/rejected_bottom_tokens": -13.9375,
      "eval_logps/rejected_top_tokens": -0.00061798095703125,
      "eval_loss": 0.584460437297821,
      "eval_rewards/accuracies": 0.6865671873092651,
      "eval_rewards/chosen": -1.1015625,
      "eval_rewards/margins": 0.5078125,
      "eval_rewards/rejected": -1.6171875,
      "eval_runtime": 107.7958,
      "eval_samples_per_second": 18.554,
      "eval_steps_per_second": 0.622,
      "step": 1200
    },
    {
      "epoch": 2.3748773307163886,
      "grad_norm": 5.209243987769614,
      "learning_rate": 4.7166358127689104e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -1.359375,
      "logps/chosen": -506.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00058746337890625,
      "logps/rejected": -510.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000614166259765625,
      "loss": 0.5627,
      "rewards/accuracies": 0.7600000500679016,
      "rewards/chosen": -0.9921875,
      "rewards/margins": 0.5390625,
      "rewards/rejected": -1.53125,
      "step": 1210
    },
    {
      "epoch": 2.39450441609421,
      "grad_norm": 4.794063744707617,
      "learning_rate": 4.708655484040467e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.34375,
      "logps/chosen": -452.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00058746337890625,
      "logps/rejected": -472.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00061798095703125,
      "loss": 0.5477,
      "rewards/accuracies": 0.7150000333786011,
      "rewards/chosen": -0.87890625,
      "rewards/margins": 0.5390625,
      "rewards/rejected": -1.421875,
      "step": 1220
    },
    {
      "epoch": 2.4141315014720313,
      "grad_norm": 5.531608721469285,
      "learning_rate": 4.7005712816512095e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.3515625,
      "logps/chosen": -438.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000591278076171875,
      "logps/rejected": -436.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.0005645751953125,
      "loss": 0.5745,
      "rewards/accuracies": 0.7199999690055847,
      "rewards/chosen": -0.78515625,
      "rewards/margins": 0.51171875,
      "rewards/rejected": -1.296875,
      "step": 1230
    },
    {
      "epoch": 2.433758586849853,
      "grad_norm": 5.267301228863608,
      "learning_rate": 4.692383585803349e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -1.34375,
      "logps/chosen": -484.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000598907470703125,
      "logps/rejected": -500.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00061798095703125,
      "loss": 0.5397,
      "rewards/accuracies": 0.7150000333786011,
      "rewards/chosen": -0.78125,
      "rewards/margins": 0.52734375,
      "rewards/rejected": -1.3046875,
      "step": 1240
    },
    {
      "epoch": 2.4533856722276743,
      "grad_norm": 5.049573801663012,
      "learning_rate": 4.6840927815664215e-07,
      "logits/chosen": -1.2578125,
      "logits/rejected": -1.3359375,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000736236572265625,
      "logps/rejected": -510.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000701904296875,
      "loss": 0.5352,
      "rewards/accuracies": 0.7350000143051147,
      "rewards/chosen": -1.0078125,
      "rewards/margins": 0.52734375,
      "rewards/rejected": -1.53125,
      "step": 1250
    },
    {
      "epoch": 2.4730127576054954,
      "grad_norm": 4.988013454206055,
      "learning_rate": 4.675699258859177e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.3515625,
      "logps/chosen": -474.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000652313232421875,
      "logps/rejected": -488.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000701904296875,
      "loss": 0.5604,
      "rewards/accuracies": 0.690000057220459,
      "rewards/chosen": -1.046875,
      "rewards/margins": 0.466796875,
      "rewards/rejected": -1.515625,
      "step": 1260
    },
    {
      "epoch": 2.492639842983317,
      "grad_norm": 4.599188572679597,
      "learning_rate": 4.667203412431244e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.3671875,
      "logps/chosen": -466.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.0006103515625,
      "logps/rejected": -450.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000621795654296875,
      "loss": 0.5489,
      "rewards/accuracies": 0.7150000333786011,
      "rewards/chosen": -0.8203125,
      "rewards/margins": 0.490234375,
      "rewards/rejected": -1.3125,
      "step": 1270
    },
    {
      "epoch": 2.5122669283611385,
      "grad_norm": 4.9531276475109065,
      "learning_rate": 4.658605641844563e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -1.3671875,
      "logps/chosen": -480.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00063323974609375,
      "logps/rejected": -466.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.0006561279296875,
      "loss": 0.5545,
      "rewards/accuracies": 0.6850000619888306,
      "rewards/chosen": -1.0234375,
      "rewards/margins": 0.404296875,
      "rewards/rejected": -1.4296875,
      "step": 1280
    },
    {
      "epoch": 2.5318940137389596,
      "grad_norm": 6.3499112776801345,
      "learning_rate": 4.6499063514545934e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.390625,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000701904296875,
      "logps/rejected": -468.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000652313232421875,
      "loss": 0.559,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.94140625,
      "rewards/margins": 0.470703125,
      "rewards/rejected": -1.4140625,
      "step": 1290
    },
    {
      "epoch": 2.551521099116781,
      "grad_norm": 5.554588430830751,
      "learning_rate": 4.6411059503913e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -1.3125,
      "logps/chosen": -436.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.0006256103515625,
      "logps/rejected": -456.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000644683837890625,
      "loss": 0.5487,
      "rewards/accuracies": 0.6800000667572021,
      "rewards/chosen": -0.9453125,
      "rewards/margins": 0.474609375,
      "rewards/rejected": -1.421875,
      "step": 1300
    },
    {
      "epoch": 2.551521099116781,
      "eval_logits/chosen": -1.34375,
      "eval_logits/rejected": -1.3828125,
      "eval_logps/chosen": -482.0,
      "eval_logps/chosen_bottom_tokens": -14.0625,
      "eval_logps/chosen_top_tokens": -0.0006561279296875,
      "eval_logps/rejected": -476.0,
      "eval_logps/rejected_bottom_tokens": -14.0,
      "eval_logps/rejected_top_tokens": -0.0006103515625,
      "eval_loss": 0.5800756812095642,
      "eval_rewards/accuracies": 0.6925373077392578,
      "eval_rewards/chosen": -0.890625,
      "eval_rewards/margins": 0.498046875,
      "eval_rewards/rejected": -1.3828125,
      "eval_runtime": 107.6532,
      "eval_samples_per_second": 18.578,
      "eval_steps_per_second": 0.622,
      "step": 1300
    },
    {
      "epoch": 2.5711481844946027,
      "grad_norm": 4.998797445392259,
      "learning_rate": 4.6322048525399093e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.359375,
      "logps/chosen": -446.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000713348388671875,
      "logps/rejected": -470.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000736236572265625,
      "loss": 0.5418,
      "rewards/accuracies": 0.7200000286102295,
      "rewards/chosen": -0.84375,
      "rewards/margins": 0.5703125,
      "rewards/rejected": -1.4140625,
      "step": 1310
    },
    {
      "epoch": 2.590775269872424,
      "grad_norm": 6.338454672487353,
      "learning_rate": 4.6232034765214444e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.3359375,
      "logps/chosen": -418.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000690460205078125,
      "logps/rejected": -452.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000690460205078125,
      "loss": 0.543,
      "rewards/accuracies": 0.7099999785423279,
      "rewards/chosen": -0.94140625,
      "rewards/margins": 0.48828125,
      "rewards/rejected": -1.4296875,
      "step": 1320
    },
    {
      "epoch": 2.6104023552502453,
      "grad_norm": 6.31080886957666,
      "learning_rate": 4.614102245673039e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.359375,
      "logps/chosen": -478.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000713348388671875,
      "logps/rejected": -496.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000698089599609375,
      "loss": 0.5482,
      "rewards/accuracies": 0.6950000524520874,
      "rewards/chosen": -1.09375,
      "rewards/margins": 0.41015625,
      "rewards/rejected": -1.5,
      "step": 1330
    },
    {
      "epoch": 2.630029440628067,
      "grad_norm": 6.076214388108869,
      "learning_rate": 4.604901588028023e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.3515625,
      "logps/chosen": -452.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0007476806640625,
      "logps/rejected": -476.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000732421875,
      "loss": 0.521,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0078125,
      "rewards/margins": 0.7109375,
      "rewards/rejected": -1.7109375,
      "step": 1340
    },
    {
      "epoch": 2.649656526005888,
      "grad_norm": 6.946264433751439,
      "learning_rate": 4.5956019362958e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.359375,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00074005126953125,
      "logps/rejected": -490.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000720977783203125,
      "loss": 0.5622,
      "rewards/accuracies": 0.6699999570846558,
      "rewards/chosen": -1.046875,
      "rewards/margins": 0.458984375,
      "rewards/rejected": -1.5078125,
      "step": 1350
    },
    {
      "epoch": 2.6692836113837095,
      "grad_norm": 5.855183962780445,
      "learning_rate": 4.586203727841488e-07,
      "logits/chosen": -1.28125,
      "logits/rejected": -1.296875,
      "logps/chosen": -424.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00064849853515625,
      "logps/rejected": -466.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000675201416015625,
      "loss": 0.5507,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8671875,
      "rewards/margins": 0.5703125,
      "rewards/rejected": -1.4375,
      "step": 1360
    },
    {
      "epoch": 2.688910696761531,
      "grad_norm": 4.854275700163788,
      "learning_rate": 4.576707404665355e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.3671875,
      "logps/chosen": -478.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.0005950927734375,
      "logps/rejected": -498.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0006103515625,
      "loss": 0.5518,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.87890625,
      "rewards/margins": 0.55859375,
      "rewards/rejected": -1.4375,
      "step": 1370
    },
    {
      "epoch": 2.708537782139352,
      "grad_norm": 4.985030119033812,
      "learning_rate": 4.5671134133820333e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -1.359375,
      "logps/chosen": -470.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000614166259765625,
      "logps/rejected": -460.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.0006103515625,
      "loss": 0.5583,
      "rewards/accuracies": 0.6899999976158142,
      "rewards/chosen": -1.0546875,
      "rewards/margins": 0.45703125,
      "rewards/rejected": -1.5078125,
      "step": 1380
    },
    {
      "epoch": 2.7281648675171737,
      "grad_norm": 4.702085035851529,
      "learning_rate": 4.5574222051995084e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -1.34375,
      "logps/chosen": -432.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000698089599609375,
      "logps/rejected": -458.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00074005126953125,
      "loss": 0.5567,
      "rewards/accuracies": 0.6800000071525574,
      "rewards/chosen": -0.90234375,
      "rewards/margins": 0.4765625,
      "rewards/rejected": -1.375,
      "step": 1390
    },
    {
      "epoch": 2.7477919528949952,
      "grad_norm": 4.845696455395146,
      "learning_rate": 4.547634235897906e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.3203125,
      "logps/chosen": -432.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00058746337890625,
      "logps/rejected": -442.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000591278076171875,
      "loss": 0.543,
      "rewards/accuracies": 0.6850000023841858,
      "rewards/chosen": -0.8125,
      "rewards/margins": 0.46484375,
      "rewards/rejected": -1.2734375,
      "step": 1400
    },
    {
      "epoch": 2.7477919528949952,
      "eval_logits/chosen": -1.3359375,
      "eval_logits/rejected": -1.375,
      "eval_logps/chosen": -480.0,
      "eval_logps/chosen_bottom_tokens": -14.0625,
      "eval_logps/chosen_top_tokens": -0.000675201416015625,
      "eval_logps/rejected": -474.0,
      "eval_logps/rejected_bottom_tokens": -13.9375,
      "eval_logps/rejected_top_tokens": -0.000606536865234375,
      "eval_loss": 0.5784887671470642,
      "eval_rewards/accuracies": 0.7134329080581665,
      "eval_rewards/chosen": -0.8671875,
      "eval_rewards/margins": 0.486328125,
      "eval_rewards/rejected": -1.3515625,
      "eval_runtime": 107.6618,
      "eval_samples_per_second": 18.577,
      "eval_steps_per_second": 0.622,
      "step": 1400
    },
    {
      "epoch": 2.7674190382728163,
      "grad_norm": 4.4890429623349855,
      "learning_rate": 4.537749965808052e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.375,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.0005645751953125,
      "logps/rejected": -512.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00054168701171875,
      "loss": 0.5466,
      "rewards/accuracies": 0.6800000667572021,
      "rewards/chosen": -0.9765625,
      "rewards/margins": 0.478515625,
      "rewards/rejected": -1.453125,
      "step": 1410
    },
    {
      "epoch": 2.787046123650638,
      "grad_norm": 6.95457084928623,
      "learning_rate": 4.527769859789825e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -1.359375,
      "logps/chosen": -464.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.0006561279296875,
      "logps/rejected": -444.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000621795654296875,
      "loss": 0.5333,
      "rewards/accuracies": 0.7099999785423279,
      "rewards/chosen": -1.03125,
      "rewards/margins": 0.5078125,
      "rewards/rejected": -1.5390625,
      "step": 1420
    },
    {
      "epoch": 2.8066732090284594,
      "grad_norm": 5.537816441709266,
      "learning_rate": 4.517694387210291e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.375,
      "logps/chosen": -484.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00069427490234375,
      "logps/rejected": -476.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00067138671875,
      "loss": 0.5315,
      "rewards/accuracies": 0.7350000739097595,
      "rewards/chosen": -1.0234375,
      "rewards/margins": 0.51171875,
      "rewards/rejected": -1.5390625,
      "step": 1430
    },
    {
      "epoch": 2.8263002944062805,
      "grad_norm": 4.999382915134583,
      "learning_rate": 4.507524021921633e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.3828125,
      "logps/chosen": -478.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00057220458984375,
      "logps/rejected": -492.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000576019287109375,
      "loss": 0.5444,
      "rewards/accuracies": 0.7049999833106995,
      "rewards/chosen": -0.9609375,
      "rewards/margins": 0.55078125,
      "rewards/rejected": -1.5078125,
      "step": 1440
    },
    {
      "epoch": 2.845927379784102,
      "grad_norm": 4.978033889471835,
      "learning_rate": 4.4972592422388635e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.3671875,
      "logps/chosen": -488.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000576019287109375,
      "logps/rejected": -484.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000560760498046875,
      "loss": 0.5154,
      "rewards/accuracies": 0.7799999713897705,
      "rewards/chosen": -0.98046875,
      "rewards/margins": 0.62890625,
      "rewards/rejected": -1.609375,
      "step": 1450
    },
    {
      "epoch": 2.8655544651619236,
      "grad_norm": 5.141801556909101,
      "learning_rate": 4.486900530917328e-07,
      "logits/chosen": -1.3125,
      "logits/rejected": -1.3046875,
      "logps/chosen": -478.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00067138671875,
      "logps/rejected": -524.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00067138671875,
      "loss": 0.5497,
      "rewards/accuracies": 0.7400000095367432,
      "rewards/chosen": -1.03125,
      "rewards/margins": 0.609375,
      "rewards/rejected": -1.640625,
      "step": 1460
    },
    {
      "epoch": 2.8851815505397447,
      "grad_norm": 5.912963606968759,
      "learning_rate": 4.4764483751300034e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.390625,
      "logps/chosen": -488.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000682830810546875,
      "logps/rejected": -478.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.00064849853515625,
      "loss": 0.5415,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -0.9765625,
      "rewards/margins": 0.578125,
      "rewards/rejected": -1.5546875,
      "step": 1470
    },
    {
      "epoch": 2.904808635917566,
      "grad_norm": 6.0601731183438625,
      "learning_rate": 4.465903266444585e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.40625,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.00066375732421875,
      "logps/rejected": -490.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.000621795654296875,
      "loss": 0.5422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0390625,
      "rewards/margins": 0.59375,
      "rewards/rejected": -1.6328125,
      "step": 1480
    },
    {
      "epoch": 2.9244357212953878,
      "grad_norm": 6.188981701284928,
      "learning_rate": 4.4552657008003666e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.3671875,
      "logps/chosen": -484.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000659942626953125,
      "logps/rejected": -524.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000667572021484375,
      "loss": 0.5532,
      "rewards/accuracies": 0.7050000429153442,
      "rewards/chosen": -1.140625,
      "rewards/margins": 0.498046875,
      "rewards/rejected": -1.6328125,
      "step": 1490
    },
    {
      "epoch": 2.944062806673209,
      "grad_norm": 5.440504391323201,
      "learning_rate": 4.444536178484919e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.3515625,
      "logps/chosen": -486.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.00051116943359375,
      "logps/rejected": -502.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000522613525390625,
      "loss": 0.5382,
      "rewards/accuracies": 0.7049999833106995,
      "rewards/chosen": -1.09375,
      "rewards/margins": 0.5078125,
      "rewards/rejected": -1.6015625,
      "step": 1500
    },
    {
      "epoch": 2.944062806673209,
      "eval_logits/chosen": -1.3515625,
      "eval_logits/rejected": -1.390625,
      "eval_logps/chosen": -506.0,
      "eval_logps/chosen_bottom_tokens": -14.125,
      "eval_logps/chosen_top_tokens": -0.000667572021484375,
      "eval_logps/rejected": -504.0,
      "eval_logps/rejected_bottom_tokens": -14.0,
      "eval_logps/rejected_top_tokens": -0.000606536865234375,
      "eval_loss": 0.5711303949356079,
      "eval_rewards/accuracies": 0.6955224275588989,
      "eval_rewards/chosen": -1.1171875,
      "eval_rewards/margins": 0.55078125,
      "eval_rewards/rejected": -1.6640625,
      "eval_runtime": 107.6549,
      "eval_samples_per_second": 18.578,
      "eval_steps_per_second": 0.622,
      "step": 1500
    },
    {
      "epoch": 2.9636898920510304,
      "grad_norm": 5.413703139778055,
      "learning_rate": 4.4337152041105597e-07,
      "logits/chosen": -1.3046875,
      "logits/rejected": -1.359375,
      "logps/chosen": -490.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000579833984375,
      "logps/rejected": -492.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.000598907470703125,
      "loss": 0.5143,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": -0.98828125,
      "rewards/margins": 0.72265625,
      "rewards/rejected": -1.7109375,
      "step": 1510
    },
    {
      "epoch": 2.983316977428852,
      "grad_norm": 6.2588549813806615,
      "learning_rate": 4.42280328659062e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.375,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000537872314453125,
      "logps/rejected": -544.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000576019287109375,
      "loss": 0.5463,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.078125,
      "rewards/margins": 0.52734375,
      "rewards/rejected": -1.609375,
      "step": 1520
    },
    {
      "epoch": 3.002944062806673,
      "grad_norm": 5.850837619696955,
      "learning_rate": 4.411800939115512e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.3984375,
      "logps/chosen": -508.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.0006256103515625,
      "logps/rejected": -476.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.000598907470703125,
      "loss": 0.5182,
      "rewards/accuracies": 0.7799999713897705,
      "rewards/chosen": -0.92578125,
      "rewards/margins": 0.60546875,
      "rewards/rejected": -1.53125,
      "step": 1530
    },
    {
      "epoch": 3.0225711481844946,
      "grad_norm": 5.1157355583646105,
      "learning_rate": 4.4007086791285955e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.375,
      "logps/chosen": -506.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000762939453125,
      "logps/rejected": -494.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000728607177734375,
      "loss": 0.5018,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.0234375,
      "rewards/margins": 0.77734375,
      "rewards/rejected": -1.8046875,
      "step": 1540
    },
    {
      "epoch": 3.042198233562316,
      "grad_norm": 5.750764600991557,
      "learning_rate": 4.3895270283018356e-07,
      "logits/chosen": -1.296875,
      "logits/rejected": -1.34375,
      "logps/chosen": -470.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.00067901611328125,
      "logps/rejected": -500.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.0006866455078125,
      "loss": 0.5185,
      "rewards/accuracies": 0.7900000810623169,
      "rewards/chosen": -0.98828125,
      "rewards/margins": 0.73046875,
      "rewards/rejected": -1.71875,
      "step": 1550
    },
    {
      "epoch": 3.061825318940137,
      "grad_norm": 6.883922300768385,
      "learning_rate": 4.3782565125112764e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.4140625,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000606536865234375,
      "logps/rejected": -500.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00061798095703125,
      "loss": 0.5135,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0390625,
      "rewards/margins": 0.57421875,
      "rewards/rejected": -1.6171875,
      "step": 1560
    },
    {
      "epoch": 3.0814524043179587,
      "grad_norm": 6.429331009978246,
      "learning_rate": 4.3668976618123035e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.3671875,
      "logps/chosen": -488.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000583648681640625,
      "logps/rejected": -524.0,
      "logps/rejected_bottom_tokens": -14.1875,
      "logps/rejected_top_tokens": -0.00060272216796875,
      "loss": 0.5316,
      "rewards/accuracies": 0.7099999785423279,
      "rewards/chosen": -1.1875,
      "rewards/margins": 0.52734375,
      "rewards/rejected": -1.71875,
      "step": 1570
    },
    {
      "epoch": 3.1010794896957803,
      "grad_norm": 5.829585261996114,
      "learning_rate": 4.3554510104147155e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.390625,
      "logps/chosen": -476.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000705718994140625,
      "logps/rejected": -502.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.0006866455078125,
      "loss": 0.5077,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.0390625,
      "rewards/margins": 0.609375,
      "rewards/rejected": -1.6484375,
      "step": 1580
    },
    {
      "epoch": 3.1207065750736014,
      "grad_norm": 7.072235636297424,
      "learning_rate": 4.3439170966576056e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.40625,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000606536865234375,
      "logps/rejected": -532.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000576019287109375,
      "loss": 0.5134,
      "rewards/accuracies": 0.7850000262260437,
      "rewards/chosen": -1.125,
      "rewards/margins": 0.640625,
      "rewards/rejected": -1.765625,
      "step": 1590
    },
    {
      "epoch": 3.140333660451423,
      "grad_norm": 6.465350023606307,
      "learning_rate": 4.332296462984034e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3828125,
      "logps/chosen": -466.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000606536865234375,
      "logps/rejected": -494.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000629425048828125,
      "loss": 0.5117,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.2265625,
      "rewards/margins": 0.58984375,
      "rewards/rejected": -1.8125,
      "step": 1600
    },
    {
      "epoch": 3.140333660451423,
      "eval_logits/chosen": -1.359375,
      "eval_logits/rejected": -1.3984375,
      "eval_logps/chosen": -520.0,
      "eval_logps/chosen_bottom_tokens": -14.125,
      "eval_logps/chosen_top_tokens": -0.00067901611328125,
      "eval_logps/rejected": -520.0,
      "eval_logps/rejected_bottom_tokens": -14.0,
      "eval_logps/rejected_top_tokens": -0.0006256103515625,
      "eval_loss": 0.5712475776672363,
      "eval_rewards/accuracies": 0.7044776082038879,
      "eval_rewards/chosen": -1.25,
      "eval_rewards/margins": 0.57421875,
      "eval_rewards/rejected": -1.828125,
      "eval_runtime": 107.7573,
      "eval_samples_per_second": 18.56,
      "eval_steps_per_second": 0.622,
      "step": 1600
    },
    {
      "epoch": 3.1599607458292445,
      "grad_norm": 6.287359311382746,
      "learning_rate": 4.3205896559155264e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.390625,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000652313232421875,
      "logps/rejected": -490.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000652313232421875,
      "loss": 0.518,
      "rewards/accuracies": 0.7849999666213989,
      "rewards/chosen": -0.94140625,
      "rewards/margins": 0.71484375,
      "rewards/rejected": -1.65625,
      "step": 1610
    },
    {
      "epoch": 3.1795878312070656,
      "grad_norm": 6.35600731483251,
      "learning_rate": 4.3087972260263636e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.390625,
      "logps/chosen": -476.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00066375732421875,
      "logps/rejected": -482.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000675201416015625,
      "loss": 0.5027,
      "rewards/accuracies": 0.7649999856948853,
      "rewards/chosen": -0.9296875,
      "rewards/margins": 0.609375,
      "rewards/rejected": -1.5390625,
      "step": 1620
    },
    {
      "epoch": 3.199214916584887,
      "grad_norm": 6.047993507610401,
      "learning_rate": 4.2969197279176917e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.3984375,
      "logps/chosen": -444.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.0005950927734375,
      "logps/rejected": -472.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00063323974609375,
      "loss": 0.515,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.91015625,
      "rewards/margins": 0.671875,
      "rewards/rejected": -1.578125,
      "step": 1630
    },
    {
      "epoch": 3.2188420019627086,
      "grad_norm": 6.629084333953029,
      "learning_rate": 4.2849577201914376e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.3671875,
      "logps/chosen": -488.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00074005126953125,
      "logps/rejected": -482.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000736236572265625,
      "loss": 0.5021,
      "rewards/accuracies": 0.7200000286102295,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 0.61328125,
      "rewards/rejected": -1.6875,
      "step": 1640
    },
    {
      "epoch": 3.2384690873405297,
      "grad_norm": 6.476913702795223,
      "learning_rate": 4.272911765424039e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -548.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000942230224609375,
      "loss": 0.5168,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.234375,
      "rewards/margins": 0.69140625,
      "rewards/rejected": -1.921875,
      "step": 1650
    },
    {
      "epoch": 3.2580961727183513,
      "grad_norm": 5.8847665430956075,
      "learning_rate": 4.2607824301399843e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -496.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00070953369140625,
      "logps/rejected": -524.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00070953369140625,
      "loss": 0.5208,
      "rewards/accuracies": 0.7800000309944153,
      "rewards/chosen": -1.078125,
      "rewards/margins": 0.65625,
      "rewards/rejected": -1.734375,
      "step": 1660
    },
    {
      "epoch": 3.277723258096173,
      "grad_norm": 6.421810533396227,
      "learning_rate": 4.2485702847851715e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.40625,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.00064849853515625,
      "logps/rejected": -552.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00070953369140625,
      "loss": 0.5022,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.0546875,
      "rewards/margins": 0.68359375,
      "rewards/rejected": -1.7421875,
      "step": 1670
    },
    {
      "epoch": 3.297350343473994,
      "grad_norm": 6.6608511509443264,
      "learning_rate": 4.2362759037000774e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.3828125,
      "logps/chosen": -484.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00067138671875,
      "logps/rejected": -510.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00067138671875,
      "loss": 0.5126,
      "rewards/accuracies": 0.7399999499320984,
      "rewards/chosen": -1.109375,
      "rewards/margins": 0.6484375,
      "rewards/rejected": -1.75,
      "step": 1680
    },
    {
      "epoch": 3.3169774288518155,
      "grad_norm": 8.22249557148097,
      "learning_rate": 4.223899865092748e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.40625,
      "logps/chosen": -474.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.00066375732421875,
      "logps/rejected": -516.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00067138671875,
      "loss": 0.5224,
      "rewards/accuracies": 0.8149999380111694,
      "rewards/chosen": -1.09375,
      "rewards/margins": 0.796875,
      "rewards/rejected": -1.890625,
      "step": 1690
    },
    {
      "epoch": 3.336604514229637,
      "grad_norm": 6.131226194897482,
      "learning_rate": 4.211442751011603e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -1.390625,
      "logps/chosen": -490.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000804901123046875,
      "logps/rejected": -516.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000774383544921875,
      "loss": 0.4983,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": -1.0078125,
      "rewards/margins": 0.7265625,
      "rewards/rejected": -1.734375,
      "step": 1700
    },
    {
      "epoch": 3.336604514229637,
      "eval_logits/chosen": -1.3671875,
      "eval_logits/rejected": -1.40625,
      "eval_logps/chosen": -510.0,
      "eval_logps/chosen_bottom_tokens": -14.125,
      "eval_logps/chosen_top_tokens": -0.00074005126953125,
      "eval_logps/rejected": -512.0,
      "eval_logps/rejected_bottom_tokens": -14.0,
      "eval_logps/rejected_top_tokens": -0.000690460205078125,
      "eval_loss": 0.5702734589576721,
      "eval_rewards/accuracies": 0.7014926075935364,
      "eval_rewards/chosen": -1.1640625,
      "eval_rewards/margins": 0.5859375,
      "eval_rewards/rejected": -1.75,
      "eval_runtime": 107.7445,
      "eval_samples_per_second": 18.562,
      "eval_steps_per_second": 0.622,
      "step": 1700
    },
    {
      "epoch": 3.356231599607458,
      "grad_norm": 6.248071111160677,
      "learning_rate": 4.198905147318065e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.390625,
      "logps/chosen": -504.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.0006866455078125,
      "logps/rejected": -532.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00067138671875,
      "loss": 0.5105,
      "rewards/accuracies": 0.7899999618530273,
      "rewards/chosen": -0.90234375,
      "rewards/margins": 0.72265625,
      "rewards/rejected": -1.625,
      "step": 1710
    },
    {
      "epoch": 3.3758586849852796,
      "grad_norm": 5.417487111723758,
      "learning_rate": 4.1862876436590045e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.4296875,
      "logps/chosen": -462.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000823974609375,
      "logps/rejected": -464.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00078582763671875,
      "loss": 0.5037,
      "rewards/accuracies": 0.7400000095367432,
      "rewards/chosen": -0.99609375,
      "rewards/margins": 0.68359375,
      "rewards/rejected": -1.6796875,
      "step": 1720
    },
    {
      "epoch": 3.395485770363101,
      "grad_norm": 6.486611676994055,
      "learning_rate": 4.173590833439008e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.40625,
      "logps/chosen": -496.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.0006561279296875,
      "logps/rejected": -482.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00066375732421875,
      "loss": 0.5146,
      "rewards/accuracies": 0.7350000143051147,
      "rewards/chosen": -1.046875,
      "rewards/margins": 0.62890625,
      "rewards/rejected": -1.6796875,
      "step": 1730
    },
    {
      "epoch": 3.4151128557409223,
      "grad_norm": 6.780283032351629,
      "learning_rate": 4.160815313792472e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.3828125,
      "logps/chosen": -468.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000743865966796875,
      "logps/rejected": -478.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000774383544921875,
      "loss": 0.5133,
      "rewards/accuracies": 0.7199999690055847,
      "rewards/chosen": -1.0546875,
      "rewards/margins": 0.5703125,
      "rewards/rejected": -1.6171875,
      "step": 1740
    },
    {
      "epoch": 3.434739941118744,
      "grad_norm": 7.370844460366542,
      "learning_rate": 4.1479616855555166e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.421875,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000736236572265625,
      "logps/rejected": -506.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000759124755859375,
      "loss": 0.529,
      "rewards/accuracies": 0.7199999690055847,
      "rewards/chosen": -1.03125,
      "rewards/margins": 0.6484375,
      "rewards/rejected": -1.6796875,
      "step": 1750
    },
    {
      "epoch": 3.4543670264965654,
      "grad_norm": 7.3760923185749645,
      "learning_rate": 4.1350305532377327e-07,
      "logits/chosen": -1.3203125,
      "logits/rejected": -1.3671875,
      "logps/chosen": -556.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000766754150390625,
      "logps/rejected": -560.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000759124755859375,
      "loss": 0.5115,
      "rewards/accuracies": 0.8050000071525574,
      "rewards/chosen": -1.21875,
      "rewards/margins": 0.8125,
      "rewards/rejected": -2.03125,
      "step": 1760
    },
    {
      "epoch": 3.4739941118743864,
      "grad_norm": 6.087167212711648,
      "learning_rate": 4.122022524993747e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -488.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000774383544921875,
      "logps/rejected": -498.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.0007476806640625,
      "loss": 0.5004,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.1640625,
      "rewards/margins": 0.6953125,
      "rewards/rejected": -1.859375,
      "step": 1770
    },
    {
      "epoch": 3.493621197252208,
      "grad_norm": 5.966209515920573,
      "learning_rate": 4.1089382125946217e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.421875,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000713348388671875,
      "logps/rejected": -504.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00072479248046875,
      "loss": 0.4945,
      "rewards/accuracies": 0.7199999690055847,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 0.703125,
      "rewards/rejected": -1.7734375,
      "step": 1780
    },
    {
      "epoch": 3.5132482826300295,
      "grad_norm": 7.519074638967278,
      "learning_rate": 4.0957782313990854e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.421875,
      "logps/chosen": -468.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000759124755859375,
      "logps/rejected": -466.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.0007476806640625,
      "loss": 0.5053,
      "rewards/accuracies": 0.7649999856948853,
      "rewards/chosen": -0.98046875,
      "rewards/margins": 0.71875,
      "rewards/rejected": -1.6953125,
      "step": 1790
    },
    {
      "epoch": 3.5328753680078506,
      "grad_norm": 6.403615147944073,
      "learning_rate": 4.0825432003245896e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.4296875,
      "logps/chosen": -502.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000640869140625,
      "logps/rejected": -506.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000640869140625,
      "loss": 0.4976,
      "rewards/accuracies": 0.7599999904632568,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 0.734375,
      "rewards/rejected": -1.8828125,
      "step": 1800
    },
    {
      "epoch": 3.5328753680078506,
      "eval_logits/chosen": -1.375,
      "eval_logits/rejected": -1.4140625,
      "eval_logps/chosen": -520.0,
      "eval_logps/chosen_bottom_tokens": -14.125,
      "eval_logps/chosen_top_tokens": -0.000732421875,
      "eval_logps/rejected": -524.0,
      "eval_logps/rejected_bottom_tokens": -14.0625,
      "eval_logps/rejected_top_tokens": -0.000690460205078125,
      "eval_loss": 0.57086181640625,
      "eval_rewards/accuracies": 0.7253731489181519,
      "eval_rewards/chosen": -1.265625,
      "eval_rewards/margins": 0.61328125,
      "eval_rewards/rejected": -1.8828125,
      "eval_runtime": 107.8127,
      "eval_samples_per_second": 18.551,
      "eval_steps_per_second": 0.621,
      "step": 1800
    },
    {
      "epoch": 3.552502453385672,
      "grad_norm": 6.030308353243273,
      "learning_rate": 4.069233741818201e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.40625,
      "logps/chosen": -490.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000759124755859375,
      "logps/rejected": -516.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000774383544921875,
      "loss": 0.4954,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.1171875,
      "rewards/margins": 0.734375,
      "rewards/rejected": -1.8515625,
      "step": 1810
    },
    {
      "epoch": 3.5721295387634937,
      "grad_norm": 6.38126467770384,
      "learning_rate": 4.0558504818273286e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -510.0,
      "logps/chosen_bottom_tokens": -14.25,
      "logps/chosen_top_tokens": -0.000659942626953125,
      "logps/rejected": -536.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.00066375732421875,
      "loss": 0.504,
      "rewards/accuracies": 0.7850000262260437,
      "rewards/chosen": -1.046875,
      "rewards/margins": 0.7734375,
      "rewards/rejected": -1.8203125,
      "step": 1820
    },
    {
      "epoch": 3.591756624141315,
      "grad_norm": 7.708115389756642,
      "learning_rate": 4.0423940497702856e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -462.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00077056884765625,
      "logps/rejected": -498.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000812530517578125,
      "loss": 0.4979,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0,
      "rewards/margins": 0.64453125,
      "rewards/rejected": -1.6484375,
      "step": 1830
    },
    {
      "epoch": 3.6113837095191363,
      "grad_norm": 6.421955849678387,
      "learning_rate": 4.028865078506688e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.3671875,
      "logps/chosen": -516.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000736236572265625,
      "logps/rejected": -544.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000759124755859375,
      "loss": 0.4938,
      "rewards/accuracies": 0.7300000786781311,
      "rewards/chosen": -1.125,
      "rewards/margins": 0.640625,
      "rewards/rejected": -1.765625,
      "step": 1840
    },
    {
      "epoch": 3.631010794896958,
      "grad_norm": 7.117336460287237,
      "learning_rate": 4.015264204307688e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -458.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00078582763671875,
      "logps/rejected": -494.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000782012939453125,
      "loss": 0.5196,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.21875,
      "rewards/margins": 0.66015625,
      "rewards/rejected": -1.875,
      "step": 1850
    },
    {
      "epoch": 3.650637880274779,
      "grad_norm": 6.582818402503727,
      "learning_rate": 4.001592066826054e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3671875,
      "logps/chosen": -478.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -532.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.00089263916015625,
      "loss": 0.5042,
      "rewards/accuracies": 0.7799999713897705,
      "rewards/chosen": -1.3125,
      "rewards/margins": 0.69140625,
      "rewards/rejected": -2.0,
      "step": 1860
    },
    {
      "epoch": 3.6702649656526005,
      "grad_norm": 8.021762341081473,
      "learning_rate": 3.987849309066085e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.3515625,
      "logps/chosen": -490.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00084686279296875,
      "logps/rejected": -512.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000812530517578125,
      "loss": 0.4992,
      "rewards/accuracies": 0.7849999666213989,
      "rewards/chosen": -1.171875,
      "rewards/margins": 0.71875,
      "rewards/rejected": -1.890625,
      "step": 1870
    },
    {
      "epoch": 3.689892051030422,
      "grad_norm": 6.8418244724657225,
      "learning_rate": 3.97403657735337e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -1.3671875,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000789642333984375,
      "logps/rejected": -532.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000789642333984375,
      "loss": 0.5026,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.1171875,
      "rewards/margins": 0.66015625,
      "rewards/rejected": -1.7734375,
      "step": 1880
    },
    {
      "epoch": 3.709519136408243,
      "grad_norm": 6.138371602438784,
      "learning_rate": 3.9601545213043936e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.34375,
      "logps/chosen": -470.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000843048095703125,
      "logps/rejected": -510.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00089263916015625,
      "loss": 0.5065,
      "rewards/accuracies": 0.7500000596046448,
      "rewards/chosen": -1.15625,
      "rewards/margins": 0.67578125,
      "rewards/rejected": -1.828125,
      "step": 1890
    },
    {
      "epoch": 3.7291462217860647,
      "grad_norm": 6.159127929581513,
      "learning_rate": 3.9462037937959815e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.390625,
      "logps/chosen": -498.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000751495361328125,
      "logps/rejected": -532.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.000759124755859375,
      "loss": 0.4956,
      "rewards/accuracies": 0.7350000143051147,
      "rewards/chosen": -1.171875,
      "rewards/margins": 0.640625,
      "rewards/rejected": -1.8203125,
      "step": 1900
    },
    {
      "epoch": 3.7291462217860647,
      "eval_logits/chosen": -1.3671875,
      "eval_logits/rejected": -1.40625,
      "eval_logps/chosen": -516.0,
      "eval_logps/chosen_bottom_tokens": -14.0625,
      "eval_logps/chosen_top_tokens": -0.000820159912109375,
      "eval_logps/rejected": -516.0,
      "eval_logps/rejected_bottom_tokens": -13.9375,
      "eval_logps/rejected_top_tokens": -0.00080108642578125,
      "eval_loss": 0.5754028558731079,
      "eval_rewards/accuracies": 0.7164179682731628,
      "eval_rewards/chosen": -1.2265625,
      "eval_rewards/margins": 0.578125,
      "eval_rewards/rejected": -1.8046875,
      "eval_runtime": 107.7159,
      "eval_samples_per_second": 18.567,
      "eval_steps_per_second": 0.622,
      "step": 1900
    },
    {
      "epoch": 3.7487733071638862,
      "grad_norm": 6.771871270407492,
      "learning_rate": 3.9321850509345944e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.3671875,
      "logps/chosen": -492.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000820159912109375,
      "logps/rejected": -516.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000812530517578125,
      "loss": 0.5049,
      "rewards/accuracies": 0.7100000381469727,
      "rewards/chosen": -1.109375,
      "rewards/margins": 0.69921875,
      "rewards/rejected": -1.8046875,
      "step": 1910
    },
    {
      "epoch": 3.7684003925417073,
      "grad_norm": 6.091441244132361,
      "learning_rate": 3.918098952025478e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.3984375,
      "logps/chosen": -474.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00079345703125,
      "logps/rejected": -482.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00079345703125,
      "loss": 0.5025,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1171875,
      "rewards/margins": 0.64453125,
      "rewards/rejected": -1.7578125,
      "step": 1920
    },
    {
      "epoch": 3.788027477919529,
      "grad_norm": 7.065260385674628,
      "learning_rate": 3.9039461595416467e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -1.3671875,
      "logps/chosen": -474.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.0008087158203125,
      "logps/rejected": -496.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00080108642578125,
      "loss": 0.5114,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.109375,
      "rewards/margins": 0.6953125,
      "rewards/rejected": -1.8046875,
      "step": 1930
    },
    {
      "epoch": 3.8076545632973504,
      "grad_norm": 7.769425748717974,
      "learning_rate": 3.8897273390927355e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -560.0,
      "logps/rejected_bottom_tokens": -14.0625,
      "logps/rejected_top_tokens": -0.0008087158203125,
      "loss": 0.499,
      "rewards/accuracies": 0.7599999904632568,
      "rewards/chosen": -1.109375,
      "rewards/margins": 0.7265625,
      "rewards/rejected": -1.8359375,
      "step": 1940
    },
    {
      "epoch": 3.8272816486751715,
      "grad_norm": 9.081400742823046,
      "learning_rate": 3.8754431593936883e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -492.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000797271728515625,
      "logps/rejected": -492.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000789642333984375,
      "loss": 0.5086,
      "rewards/accuracies": 0.7649999856948853,
      "rewards/chosen": -1.0546875,
      "rewards/margins": 0.60546875,
      "rewards/rejected": -1.6640625,
      "step": 1950
    },
    {
      "epoch": 3.846908734052993,
      "grad_norm": 6.83623776664128,
      "learning_rate": 3.8610942922333157e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.375,
      "logps/chosen": -548.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000820159912109375,
      "logps/rejected": -556.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000827789306640625,
      "loss": 0.515,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 0.6875,
      "rewards/rejected": -1.8359375,
      "step": 1960
    },
    {
      "epoch": 3.8665358194308146,
      "grad_norm": 6.2883991743173775,
      "learning_rate": 3.8466814124426937e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.3671875,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000843048095703125,
      "logps/rejected": -536.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0008697509765625,
      "loss": 0.4939,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 0.66796875,
      "rewards/rejected": -2.0,
      "step": 1970
    },
    {
      "epoch": 3.8861629048086357,
      "grad_norm": 6.696772904643436,
      "learning_rate": 3.832205197863432e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -492.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000766754150390625,
      "logps/rejected": -498.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.0007781982421875,
      "loss": 0.5156,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.265625,
      "rewards/margins": 0.625,
      "rewards/rejected": -1.890625,
      "step": 1980
    },
    {
      "epoch": 3.9057899901864572,
      "grad_norm": 6.274826409211275,
      "learning_rate": 3.817666329315792e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4453125,
      "logps/chosen": -486.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00074005126953125,
      "logps/rejected": -482.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00078582763671875,
      "loss": 0.4877,
      "rewards/accuracies": 0.7950000166893005,
      "rewards/chosen": -1.046875,
      "rewards/margins": 0.8046875,
      "rewards/rejected": -1.8515625,
      "step": 1990
    },
    {
      "epoch": 3.9254170755642788,
      "grad_norm": 7.773042696653781,
      "learning_rate": 3.8030654905666663e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.40625,
      "logps/chosen": -508.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00079345703125,
      "logps/rejected": -520.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000782012939453125,
      "loss": 0.4996,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.203125,
      "rewards/margins": 0.70703125,
      "rewards/rejected": -1.90625,
      "step": 2000
    },
    {
      "epoch": 3.9254170755642788,
      "eval_logits/chosen": -1.375,
      "eval_logits/rejected": -1.40625,
      "eval_logps/chosen": -520.0,
      "eval_logps/chosen_bottom_tokens": -14.0625,
      "eval_logps/chosen_top_tokens": -0.0008087158203125,
      "eval_logps/rejected": -524.0,
      "eval_logps/rejected_bottom_tokens": -13.9375,
      "eval_logps/rejected_top_tokens": -0.0007781982421875,
      "eval_loss": 0.5721948146820068,
      "eval_rewards/accuracies": 0.7044775485992432,
      "eval_rewards/chosen": -1.2578125,
      "eval_rewards/margins": 0.6015625,
      "eval_rewards/rejected": -1.8515625,
      "eval_runtime": 107.7979,
      "eval_samples_per_second": 18.553,
      "eval_steps_per_second": 0.622,
      "step": 2000
    },
    {
      "epoch": 3.9450441609421,
      "grad_norm": 6.221166448815382,
      "learning_rate": 3.788403368297426e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.40625,
      "logps/chosen": -488.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00083160400390625,
      "logps/rejected": -532.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00089263916015625,
      "loss": 0.4979,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": -1.1640625,
      "rewards/margins": 0.765625,
      "rewards/rejected": -1.9296875,
      "step": 2010
    },
    {
      "epoch": 3.9646712463199214,
      "grad_norm": 6.881681342957275,
      "learning_rate": 3.7736806520716194e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.359375,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0008544921875,
      "logps/rejected": -524.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000865936279296875,
      "loss": 0.4873,
      "rewards/accuracies": 0.7699999213218689,
      "rewards/chosen": -1.1015625,
      "rewards/margins": 0.69921875,
      "rewards/rejected": -1.8046875,
      "step": 2020
    },
    {
      "epoch": 3.984298331697743,
      "grad_norm": 7.245729337451655,
      "learning_rate": 3.7588980343025466e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.375,
      "logps/chosen": -508.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00084686279296875,
      "logps/rejected": -548.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000843048095703125,
      "loss": 0.486,
      "rewards/accuracies": 0.7700000405311584,
      "rewards/chosen": -1.25,
      "rewards/margins": 0.81640625,
      "rewards/rejected": -2.0625,
      "step": 2030
    },
    {
      "epoch": 4.003925417075564,
      "grad_norm": 7.746412669207162,
      "learning_rate": 3.744056210220692e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.3828125,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000919342041015625,
      "logps/rejected": -548.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.4992,
      "rewards/accuracies": 0.7300000190734863,
      "rewards/chosen": -1.4140625,
      "rewards/margins": 0.6015625,
      "rewards/rejected": -2.015625,
      "step": 2040
    },
    {
      "epoch": 4.023552502453386,
      "grad_norm": 6.945255988635086,
      "learning_rate": 3.7291558778410307e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.421875,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000823974609375,
      "logps/rejected": -510.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0008087158203125,
      "loss": 0.4725,
      "rewards/accuracies": 0.7649999856948853,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 0.72265625,
      "rewards/rejected": -1.8671875,
      "step": 2050
    },
    {
      "epoch": 4.043179587831207,
      "grad_norm": 6.647028975170552,
      "learning_rate": 3.714197737930199e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.34375,
      "logps/chosen": -458.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000827789306640625,
      "logps/rejected": -520.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000827789306640625,
      "loss": 0.4934,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": -1.125,
      "rewards/margins": 0.64453125,
      "rewards/rejected": -1.765625,
      "step": 2060
    },
    {
      "epoch": 4.062806673209028,
      "grad_norm": 7.140041695807948,
      "learning_rate": 3.699182493973532e-07,
      "logits/chosen": -1.328125,
      "logits/rejected": -1.3828125,
      "logps/chosen": -572.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000873565673828125,
      "logps/rejected": -548.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0008087158203125,
      "loss": 0.4763,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.2421875,
      "rewards/margins": 0.66796875,
      "rewards/rejected": -1.90625,
      "step": 2070
    },
    {
      "epoch": 4.08243375858685,
      "grad_norm": 7.0413962534044945,
      "learning_rate": 3.6841108521419903e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.390625,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00081634521484375,
      "logps/rejected": -560.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000774383544921875,
      "loss": 0.472,
      "rewards/accuracies": 0.7849999666213989,
      "rewards/chosen": -1.2109375,
      "rewards/margins": 0.796875,
      "rewards/rejected": -2.0,
      "step": 2080
    },
    {
      "epoch": 4.102060843964671,
      "grad_norm": 7.852773860149295,
      "learning_rate": 3.6689835212589375e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4375,
      "logps/chosen": -488.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000835418701171875,
      "logps/rejected": -480.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000827789306640625,
      "loss": 0.4699,
      "rewards/accuracies": 0.7350000143051147,
      "rewards/chosen": -1.234375,
      "rewards/margins": 0.6796875,
      "rewards/rejected": -1.9140625,
      "step": 2090
    },
    {
      "epoch": 4.121687929342492,
      "grad_norm": 7.332304315384899,
      "learning_rate": 3.6538012127668095e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.375,
      "logps/chosen": -510.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -548.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000873565673828125,
      "loss": 0.4588,
      "rewards/accuracies": 0.85999995470047,
      "rewards/chosen": -1.1953125,
      "rewards/margins": 0.9140625,
      "rewards/rejected": -2.109375,
      "step": 2100
    },
    {
      "epoch": 4.121687929342492,
      "eval_logits/chosen": -1.375,
      "eval_logits/rejected": -1.40625,
      "eval_logps/chosen": -536.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.00089263916015625,
      "eval_logps/rejected": -540.0,
      "eval_logps/rejected_bottom_tokens": -13.875,
      "eval_logps/rejected_top_tokens": -0.000873565673828125,
      "eval_loss": 0.5748266577720642,
      "eval_rewards/accuracies": 0.7343283891677856,
      "eval_rewards/chosen": -1.4140625,
      "eval_rewards/margins": 0.62109375,
      "eval_rewards/rejected": -2.03125,
      "eval_runtime": 107.7433,
      "eval_samples_per_second": 18.563,
      "eval_steps_per_second": 0.622,
      "step": 2100
    },
    {
      "epoch": 4.141315014720314,
      "grad_norm": 8.178777396954505,
      "learning_rate": 3.638564640693654e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.375,
      "logps/chosen": -490.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.0009002685546875,
      "logps/rejected": -528.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00095367431640625,
      "loss": 0.4787,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": -1.40625,
      "rewards/margins": 0.8359375,
      "rewards/rejected": -2.234375,
      "step": 2110
    },
    {
      "epoch": 4.1609421000981355,
      "grad_norm": 8.270420911364823,
      "learning_rate": 3.623274521619549e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3828125,
      "logps/chosen": -528.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00086212158203125,
      "logps/rejected": -568.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000850677490234375,
      "loss": 0.4694,
      "rewards/accuracies": 0.7899999618530273,
      "rewards/chosen": -1.2265625,
      "rewards/margins": 0.796875,
      "rewards/rejected": -2.015625,
      "step": 2120
    },
    {
      "epoch": 4.180569185475957,
      "grad_norm": 8.51762407281088,
      "learning_rate": 3.6079315746429016e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.40625,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00086212158203125,
      "logps/rejected": -568.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000873565673828125,
      "loss": 0.4691,
      "rewards/accuracies": 0.7900000214576721,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 0.87109375,
      "rewards/rejected": -2.125,
      "step": 2130
    },
    {
      "epoch": 4.200196270853779,
      "grad_norm": 7.650781283295743,
      "learning_rate": 3.592536521346631e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.375,
      "logps/chosen": -572.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00086212158203125,
      "logps/rejected": -584.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000873565673828125,
      "loss": 0.4688,
      "rewards/accuracies": 0.7649999856948853,
      "rewards/chosen": -1.1875,
      "rewards/margins": 0.765625,
      "rewards/rejected": -1.953125,
      "step": 2140
    },
    {
      "epoch": 4.2198233562316,
      "grad_norm": 8.230508945281315,
      "learning_rate": 3.5770900857642307e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.375,
      "logps/chosen": -478.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000946044921875,
      "logps/rejected": -536.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00096893310546875,
      "loss": 0.4624,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": -1.25,
      "rewards/margins": 0.80859375,
      "rewards/rejected": -2.0625,
      "step": 2150
    },
    {
      "epoch": 4.239450441609421,
      "grad_norm": 7.156745919336785,
      "learning_rate": 3.5615929943457145e-07,
      "logits/chosen": -1.34375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -516.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000904083251953125,
      "logps/rejected": -528.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000911712646484375,
      "loss": 0.4614,
      "rewards/accuracies": 0.7799999713897705,
      "rewards/chosen": -1.28125,
      "rewards/margins": 0.8359375,
      "rewards/rejected": -2.109375,
      "step": 2160
    },
    {
      "epoch": 4.259077526987243,
      "grad_norm": 8.239581657287577,
      "learning_rate": 3.546045975923457e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.375,
      "logps/chosen": -472.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -536.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000911712646484375,
      "loss": 0.4676,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": -1.125,
      "rewards/margins": 0.9453125,
      "rewards/rejected": -2.0625,
      "step": 2170
    },
    {
      "epoch": 4.278704612365064,
      "grad_norm": 9.9311762081506,
      "learning_rate": 3.530449761677911e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.3984375,
      "logps/chosen": -472.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000843048095703125,
      "logps/rejected": -490.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0008392333984375,
      "loss": 0.474,
      "rewards/accuracies": 0.7799999713897705,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 0.71484375,
      "rewards/rejected": -1.859375,
      "step": 2180
    },
    {
      "epoch": 4.298331697742885,
      "grad_norm": 7.430327144765855,
      "learning_rate": 3.5148050851032217e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.421875,
      "logps/chosen": -516.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000946044921875,
      "logps/rejected": -544.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000965118408203125,
      "loss": 0.4697,
      "rewards/accuracies": 0.7800000309944153,
      "rewards/chosen": -1.296875,
      "rewards/margins": 0.80078125,
      "rewards/rejected": -2.09375,
      "step": 2190
    },
    {
      "epoch": 4.317958783120707,
      "grad_norm": 8.411774199817168,
      "learning_rate": 3.499112681972733e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.40625,
      "logps/chosen": -512.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00091552734375,
      "logps/rejected": -536.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000904083251953125,
      "loss": 0.4555,
      "rewards/accuracies": 0.7949999570846558,
      "rewards/chosen": -1.1640625,
      "rewards/margins": 0.859375,
      "rewards/rejected": -2.03125,
      "step": 2200
    },
    {
      "epoch": 4.317958783120707,
      "eval_logits/chosen": -1.390625,
      "eval_logits/rejected": -1.421875,
      "eval_logps/chosen": -524.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.00087738037109375,
      "eval_logps/rejected": -528.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.00087738037109375,
      "eval_loss": 0.5742602348327637,
      "eval_rewards/accuracies": 0.7164179086685181,
      "eval_rewards/chosen": -1.296875,
      "eval_rewards/margins": 0.6171875,
      "eval_rewards/rejected": -1.9140625,
      "eval_runtime": 107.7452,
      "eval_samples_per_second": 18.562,
      "eval_steps_per_second": 0.622,
      "step": 2200
    },
    {
      "epoch": 4.337585868498528,
      "grad_norm": 8.874627070136235,
      "learning_rate": 3.48337329030438e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0008697509765625,
      "logps/rejected": -576.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00086212158203125,
      "loss": 0.4699,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4140625,
      "rewards/margins": 0.76171875,
      "rewards/rejected": -2.171875,
      "step": 2210
    },
    {
      "epoch": 4.357212953876349,
      "grad_norm": 9.385129625994237,
      "learning_rate": 3.467587650325981e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -450.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.0008544921875,
      "logps/rejected": -500.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000865936279296875,
      "loss": 0.4638,
      "rewards/accuracies": 0.7649999856948853,
      "rewards/chosen": -1.3125,
      "rewards/margins": 0.74609375,
      "rewards/rejected": -2.0625,
      "step": 2220
    },
    {
      "epoch": 4.376840039254171,
      "grad_norm": 7.409577352141209,
      "learning_rate": 3.4517565044404264e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4375,
      "logps/chosen": -496.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00081634521484375,
      "logps/rejected": -556.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000782012939453125,
      "loss": 0.4584,
      "rewards/accuracies": 0.8300000429153442,
      "rewards/chosen": -1.1640625,
      "rewards/margins": 0.9296875,
      "rewards/rejected": -2.09375,
      "step": 2230
    },
    {
      "epoch": 4.396467124631992,
      "grad_norm": 7.070880799909722,
      "learning_rate": 3.435880597190759e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.390625,
      "logps/chosen": -506.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0009002685546875,
      "logps/rejected": -548.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000911712646484375,
      "loss": 0.4448,
      "rewards/accuracies": 0.8349999189376831,
      "rewards/chosen": -1.34375,
      "rewards/margins": 0.94921875,
      "rewards/rejected": -2.296875,
      "step": 2240
    },
    {
      "epoch": 4.416094210009813,
      "grad_norm": 7.817845737280103,
      "learning_rate": 3.419960675225163e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.40625,
      "logps/chosen": -472.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000926971435546875,
      "logps/rejected": -536.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0009765625,
      "loss": 0.4482,
      "rewards/accuracies": 0.8100000619888306,
      "rewards/chosen": -1.3046875,
      "rewards/margins": 0.859375,
      "rewards/rejected": -2.15625,
      "step": 2250
    },
    {
      "epoch": 4.435721295387635,
      "grad_norm": 6.862127053903812,
      "learning_rate": 3.403997487261846e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.40625,
      "logps/chosen": -492.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000911712646484375,
      "logps/rejected": -556.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00092315673828125,
      "loss": 0.4831,
      "rewards/accuracies": 0.7800000309944153,
      "rewards/chosen": -1.171875,
      "rewards/margins": 0.8515625,
      "rewards/rejected": -2.015625,
      "step": 2260
    },
    {
      "epoch": 4.455348380765456,
      "grad_norm": 7.719830836324066,
      "learning_rate": 3.3879917840538264e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.390625,
      "logps/chosen": -462.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000904083251953125,
      "logps/rejected": -516.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000926971435546875,
      "loss": 0.4666,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.1171875,
      "rewards/margins": 0.78515625,
      "rewards/rejected": -1.8984375,
      "step": 2270
    },
    {
      "epoch": 4.4749754661432775,
      "grad_norm": 7.285396802308617,
      "learning_rate": 3.3719443183536263e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.390625,
      "logps/chosen": -488.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000873565673828125,
      "logps/rejected": -512.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000896453857421875,
      "loss": 0.4662,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 0.796875,
      "rewards/rejected": -1.9375,
      "step": 2280
    },
    {
      "epoch": 4.494602551521099,
      "grad_norm": 8.596136962950057,
      "learning_rate": 3.3558558448778687e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -516.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -552.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.0008087158203125,
      "loss": 0.474,
      "rewards/accuracies": 0.7150000333786011,
      "rewards/chosen": -1.203125,
      "rewards/margins": 0.71484375,
      "rewards/rejected": -1.9140625,
      "step": 2290
    },
    {
      "epoch": 4.5142296368989205,
      "grad_norm": 6.896991963163053,
      "learning_rate": 3.339727120271783e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.3515625,
      "logps/chosen": -460.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -524.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00090789794921875,
      "loss": 0.4625,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2890625,
      "rewards/margins": 0.8359375,
      "rewards/rejected": -2.125,
      "step": 2300
    },
    {
      "epoch": 4.5142296368989205,
      "eval_logits/chosen": -1.3828125,
      "eval_logits/rejected": -1.4140625,
      "eval_logps/chosen": -524.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.0008392333984375,
      "eval_logps/rejected": -532.0,
      "eval_logps/rejected_bottom_tokens": -13.875,
      "eval_logps/rejected_top_tokens": -0.00083160400390625,
      "eval_loss": 0.5734838843345642,
      "eval_rewards/accuracies": 0.7134328484535217,
      "eval_rewards/chosen": -1.3046875,
      "eval_rewards/margins": 0.625,
      "eval_rewards/rejected": -1.9296875,
      "eval_runtime": 107.7862,
      "eval_samples_per_second": 18.555,
      "eval_steps_per_second": 0.622,
      "step": 2300
    },
    {
      "epoch": 4.533856722276742,
      "grad_norm": 8.088656592962153,
      "learning_rate": 3.323558903073623e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.3984375,
      "logps/chosen": -498.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00090789794921875,
      "logps/rejected": -536.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00089263916015625,
      "loss": 0.4622,
      "rewards/accuracies": 0.7800000309944153,
      "rewards/chosen": -1.21875,
      "rewards/margins": 0.80859375,
      "rewards/rejected": -2.03125,
      "step": 2310
    },
    {
      "epoch": 4.553483807654564,
      "grad_norm": 8.835212725754523,
      "learning_rate": 3.307351953678985e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.421875,
      "logps/chosen": -504.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000949859619140625,
      "logps/rejected": -496.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000904083251953125,
      "loss": 0.4653,
      "rewards/accuracies": 0.7599999904632568,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 0.703125,
      "rewards/rejected": -2.15625,
      "step": 2320
    },
    {
      "epoch": 4.573110893032385,
      "grad_norm": 8.83751486846289,
      "learning_rate": 3.291107034305055e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -502.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00089263916015625,
      "logps/rejected": -536.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000888824462890625,
      "loss": 0.4487,
      "rewards/accuracies": 0.8050000071525574,
      "rewards/chosen": -1.3515625,
      "rewards/margins": 0.96484375,
      "rewards/rejected": -2.3125,
      "step": 2330
    },
    {
      "epoch": 4.592737978410206,
      "grad_norm": 7.538041568997226,
      "learning_rate": 3.2748249089547555e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.3359375,
      "logps/chosen": -504.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000888824462890625,
      "logps/rejected": -548.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00093841552734375,
      "loss": 0.4564,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 0.71484375,
      "rewards/rejected": -2.15625,
      "step": 2340
    },
    {
      "epoch": 4.612365063788028,
      "grad_norm": 7.102943658324035,
      "learning_rate": 3.2585063433808144e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.3828125,
      "logps/chosen": -516.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00096893310546875,
      "logps/rejected": -544.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000934600830078125,
      "loss": 0.4759,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.25,
      "rewards/margins": 0.7734375,
      "rewards/rejected": -2.015625,
      "step": 2350
    },
    {
      "epoch": 4.631992149165849,
      "grad_norm": 7.911197835666227,
      "learning_rate": 3.242152105049758e-07,
      "logits/chosen": -1.3359375,
      "logits/rejected": -1.375,
      "logps/chosen": -498.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00083160400390625,
      "logps/rejected": -556.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000835418701171875,
      "loss": 0.4776,
      "rewards/accuracies": 0.8349999189376831,
      "rewards/chosen": -1.234375,
      "rewards/margins": 0.84375,
      "rewards/rejected": -2.078125,
      "step": 2360
    },
    {
      "epoch": 4.65161923454367,
      "grad_norm": 8.320854595644885,
      "learning_rate": 3.2257629631058065e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -486.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.000885009765625,
      "logps/rejected": -524.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000843048095703125,
      "loss": 0.4497,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 0.81640625,
      "rewards/rejected": -2.15625,
      "step": 2370
    },
    {
      "epoch": 4.671246319921492,
      "grad_norm": 7.929898566219126,
      "learning_rate": 3.20933968833471e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.40625,
      "logps/chosen": -516.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -544.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000881195068359375,
      "loss": 0.4584,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 0.77734375,
      "rewards/rejected": -2.171875,
      "step": 2380
    },
    {
      "epoch": 4.690873405299313,
      "grad_norm": 7.163855496034465,
      "learning_rate": 3.1928830531274933e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -492.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -540.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000858306884765625,
      "loss": 0.4592,
      "rewards/accuracies": 0.8350000381469727,
      "rewards/chosen": -1.2734375,
      "rewards/margins": 0.9375,
      "rewards/rejected": -2.203125,
      "step": 2390
    },
    {
      "epoch": 4.710500490677134,
      "grad_norm": 7.977348623085118,
      "learning_rate": 3.176393831444131e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.3828125,
      "logps/chosen": -486.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00092315673828125,
      "logps/rejected": -564.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00093841552734375,
      "loss": 0.469,
      "rewards/accuracies": 0.7949999570846558,
      "rewards/chosen": -1.296875,
      "rewards/margins": 0.80859375,
      "rewards/rejected": -2.109375,
      "step": 2400
    },
    {
      "epoch": 4.710500490677134,
      "eval_logits/chosen": -1.3984375,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -540.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.000873565673828125,
      "eval_logps/rejected": -552.0,
      "eval_logps/rejected_bottom_tokens": -13.875,
      "eval_logps/rejected_top_tokens": -0.000858306884765625,
      "eval_loss": 0.5743066668510437,
      "eval_rewards/accuracies": 0.7194029688835144,
      "eval_rewards/chosen": -1.4765625,
      "eval_rewards/margins": 0.65625,
      "eval_rewards/rejected": -2.140625,
      "eval_runtime": 107.9291,
      "eval_samples_per_second": 18.531,
      "eval_steps_per_second": 0.621,
      "step": 2400
    },
    {
      "epoch": 4.730127576054956,
      "grad_norm": 7.459232232628322,
      "learning_rate": 3.1598727987771485e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.453125,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00080108642578125,
      "logps/rejected": -564.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00079345703125,
      "loss": 0.4722,
      "rewards/accuracies": 0.7400000095367432,
      "rewards/chosen": -1.4140625,
      "rewards/margins": 0.8203125,
      "rewards/rejected": -2.234375,
      "step": 2410
    },
    {
      "epoch": 4.749754661432777,
      "grad_norm": 8.489469253151096,
      "learning_rate": 3.1433207321151523e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.421875,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000850677490234375,
      "logps/rejected": -560.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000827789306640625,
      "loss": 0.4536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.390625,
      "rewards/margins": 0.8515625,
      "rewards/rejected": -2.234375,
      "step": 2420
    },
    {
      "epoch": 4.769381746810598,
      "grad_norm": 8.237701367212086,
      "learning_rate": 3.126738409906284e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.40625,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.0008087158203125,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00081634521484375,
      "loss": 0.434,
      "rewards/accuracies": 0.8450000882148743,
      "rewards/chosen": -1.390625,
      "rewards/margins": 0.9921875,
      "rewards/rejected": -2.390625,
      "step": 2430
    },
    {
      "epoch": 4.78900883218842,
      "grad_norm": 7.925340457533995,
      "learning_rate": 3.1101266120216124e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.4140625,
      "logps/chosen": -504.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.0008544921875,
      "logps/rejected": -556.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000858306884765625,
      "loss": 0.4718,
      "rewards/accuracies": 0.7449999451637268,
      "rewards/chosen": -1.2734375,
      "rewards/margins": 0.84765625,
      "rewards/rejected": -2.125,
      "step": 2440
    },
    {
      "epoch": 4.808635917566241,
      "grad_norm": 8.462699743036197,
      "learning_rate": 3.0934861197184547e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.4296875,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000904083251953125,
      "logps/rejected": -520.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000904083251953125,
      "loss": 0.4668,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.359375,
      "rewards/margins": 0.8046875,
      "rewards/rejected": -2.15625,
      "step": 2450
    },
    {
      "epoch": 4.8282630029440625,
      "grad_norm": 8.150345327512246,
      "learning_rate": 3.076817715603634e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.4453125,
      "logps/chosen": -476.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.00093841552734375,
      "logps/rejected": -544.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00093841552734375,
      "loss": 0.4652,
      "rewards/accuracies": 0.7800000309944153,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 0.92578125,
      "rewards/rejected": -2.328125,
      "step": 2460
    },
    {
      "epoch": 4.8478900883218845,
      "grad_norm": 8.735080643523695,
      "learning_rate": 3.060122183596676e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.421875,
      "logps/chosen": -498.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00086212158203125,
      "logps/rejected": -524.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00086212158203125,
      "loss": 0.4457,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": -1.28125,
      "rewards/margins": 0.9609375,
      "rewards/rejected": -2.25,
      "step": 2470
    },
    {
      "epoch": 4.867517173699706,
      "grad_norm": 10.381821722696003,
      "learning_rate": 3.0434003088929347e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00096893310546875,
      "logps/rejected": -560.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0009613037109375,
      "loss": 0.4586,
      "rewards/accuracies": 0.7750000357627869,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 0.921875,
      "rewards/rejected": -2.375,
      "step": 2480
    },
    {
      "epoch": 4.887144259077527,
      "grad_norm": 7.745597200658498,
      "learning_rate": 3.026652877926672e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00087738037109375,
      "logps/rejected": -540.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000850677490234375,
      "loss": 0.4611,
      "rewards/accuracies": 0.8050000071525574,
      "rewards/chosen": -1.28125,
      "rewards/margins": 0.86328125,
      "rewards/rejected": -2.140625,
      "step": 2490
    },
    {
      "epoch": 4.906771344455349,
      "grad_norm": 9.65042560567993,
      "learning_rate": 3.0098806783340644e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.3828125,
      "logps/chosen": -482.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.0008087158203125,
      "logps/rejected": -568.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00083160400390625,
      "loss": 0.4796,
      "rewards/accuracies": 0.7800000309944153,
      "rewards/chosen": -1.1796875,
      "rewards/margins": 0.96484375,
      "rewards/rejected": -2.140625,
      "step": 2500
    },
    {
      "epoch": 4.906771344455349,
      "eval_logits/chosen": -1.3984375,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -528.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.00086212158203125,
      "eval_logps/rejected": -536.0,
      "eval_logps/rejected_bottom_tokens": -13.875,
      "eval_logps/rejected_top_tokens": -0.000858306884765625,
      "eval_loss": 0.574963390827179,
      "eval_rewards/accuracies": 0.7134328484535217,
      "eval_rewards/chosen": -1.328125,
      "eval_rewards/margins": 0.6484375,
      "eval_rewards/rejected": -1.9765625,
      "eval_runtime": 107.6921,
      "eval_samples_per_second": 18.571,
      "eval_steps_per_second": 0.622,
      "step": 2500
    },
    {
      "epoch": 4.92639842983317,
      "grad_norm": 9.041272943836647,
      "learning_rate": 2.9930844989161646e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -492.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00086212158203125,
      "logps/rejected": -536.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.0008392333984375,
      "loss": 0.4643,
      "rewards/accuracies": 0.7549999952316284,
      "rewards/chosen": -1.2421875,
      "rewards/margins": 0.83984375,
      "rewards/rejected": -2.078125,
      "step": 2510
    },
    {
      "epoch": 4.946025515210991,
      "grad_norm": 8.247379866929867,
      "learning_rate": 2.9762651296018045e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -564.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.0007781982421875,
      "logps/rejected": -588.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000751495361328125,
      "loss": 0.4637,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -2.28125,
      "step": 2520
    },
    {
      "epoch": 4.965652600588813,
      "grad_norm": 7.265609055624016,
      "learning_rate": 2.9594233614104405e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.40625,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000827789306640625,
      "logps/rejected": -572.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00083160400390625,
      "loss": 0.459,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2265625,
      "rewards/margins": 0.93359375,
      "rewards/rejected": -2.15625,
      "step": 2530
    },
    {
      "epoch": 4.985279685966634,
      "grad_norm": 8.284039428637186,
      "learning_rate": 2.942559986414957e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -508.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0008697509765625,
      "logps/rejected": -556.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000873565673828125,
      "loss": 0.4378,
      "rewards/accuracies": 0.8300000429153442,
      "rewards/chosen": -1.234375,
      "rewards/margins": 0.90625,
      "rewards/rejected": -2.140625,
      "step": 2540
    },
    {
      "epoch": 5.004906771344455,
      "grad_norm": 7.587696700934878,
      "learning_rate": 2.9256757977044104e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -528.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -580.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0008544921875,
      "loss": 0.4535,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.21875,
      "rewards/margins": 0.84375,
      "rewards/rejected": -2.0625,
      "step": 2550
    },
    {
      "epoch": 5.024533856722277,
      "grad_norm": 9.079365868102483,
      "learning_rate": 2.9087715893467305e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4375,
      "logps/chosen": -564.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000843048095703125,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000823974609375,
      "loss": 0.4135,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.3515625,
      "rewards/margins": 0.92578125,
      "rewards/rejected": -2.28125,
      "step": 2560
    },
    {
      "epoch": 5.044160942100098,
      "grad_norm": 8.135351090751177,
      "learning_rate": 2.8918481563513796e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4453125,
      "logps/chosen": -528.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -560.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000881195068359375,
      "loss": 0.4309,
      "rewards/accuracies": 0.8050000071525574,
      "rewards/chosen": -1.34375,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -2.359375,
      "step": 2570
    },
    {
      "epoch": 5.063788027477919,
      "grad_norm": 9.685008220153263,
      "learning_rate": 2.874906294631957e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.453125,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00089263916015625,
      "logps/rejected": -564.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00087738037109375,
      "loss": 0.4334,
      "rewards/accuracies": 0.8299999237060547,
      "rewards/chosen": -1.40625,
      "rewards/margins": 0.9453125,
      "rewards/rejected": -2.34375,
      "step": 2580
    },
    {
      "epoch": 5.083415112855741,
      "grad_norm": 8.303566254927492,
      "learning_rate": 2.857946800968773e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -494.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -560.0,
      "logps/rejected_bottom_tokens": -14.125,
      "logps/rejected_top_tokens": -0.00089263916015625,
      "loss": 0.4368,
      "rewards/accuracies": 0.7549999952316284,
      "rewards/chosen": -1.40625,
      "rewards/margins": 0.7734375,
      "rewards/rejected": -2.1875,
      "step": 2590
    },
    {
      "epoch": 5.103042198233562,
      "grad_norm": 8.84280428313552,
      "learning_rate": 2.840970472971369e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -580.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000820159912109375,
      "loss": 0.4082,
      "rewards/accuracies": 0.8199999928474426,
      "rewards/chosen": -1.4140625,
      "rewards/margins": 1.015625,
      "rewards/rejected": -2.421875,
      "step": 2600
    },
    {
      "epoch": 5.103042198233562,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4453125,
      "eval_logps/chosen": -552.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.000904083251953125,
      "eval_logps/rejected": -564.0,
      "eval_logps/rejected_bottom_tokens": -13.875,
      "eval_logps/rejected_top_tokens": -0.0009002685546875,
      "eval_loss": 0.5817602276802063,
      "eval_rewards/accuracies": 0.7194029688835144,
      "eval_rewards/chosen": -1.6015625,
      "eval_rewards/margins": 0.66015625,
      "eval_rewards/rejected": -2.265625,
      "eval_runtime": 107.701,
      "eval_samples_per_second": 18.57,
      "eval_steps_per_second": 0.622,
      "step": 2600
    },
    {
      "epoch": 5.122669283611383,
      "grad_norm": 8.9514089823427,
      "learning_rate": 2.823978109041013e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.3828125,
      "logps/chosen": -470.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00098419189453125,
      "logps/rejected": -548.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.001007080078125,
      "loss": 0.4238,
      "rewards/accuracies": 0.8300000429153442,
      "rewards/chosen": -1.40625,
      "rewards/margins": 0.98828125,
      "rewards/rejected": -2.390625,
      "step": 2610
    },
    {
      "epoch": 5.142296368989205,
      "grad_norm": 8.78349947961107,
      "learning_rate": 2.8069705083331457e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4375,
      "logps/chosen": -494.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0009613037109375,
      "logps/rejected": -544.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0009613037109375,
      "loss": 0.4275,
      "rewards/accuracies": 0.7899999618530273,
      "rewards/chosen": -1.46875,
      "rewards/margins": 1.015625,
      "rewards/rejected": -2.484375,
      "step": 2620
    },
    {
      "epoch": 5.1619234543670265,
      "grad_norm": 9.596094631437214,
      "learning_rate": 2.789948470719798e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -14.125,
      "logps/chosen_top_tokens": -0.00092315673828125,
      "logps/rejected": -572.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000946044921875,
      "loss": 0.4266,
      "rewards/accuracies": 0.8100000619888306,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 0.96484375,
      "rewards/rejected": -2.359375,
      "step": 2630
    },
    {
      "epoch": 5.181550539744848,
      "grad_norm": 13.34951502784971,
      "learning_rate": 2.7729127967519717e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -528.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00084686279296875,
      "logps/rejected": -580.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000858306884765625,
      "loss": 0.4362,
      "rewards/accuracies": 0.7949999570846558,
      "rewards/chosen": -1.53125,
      "rewards/margins": 0.84375,
      "rewards/rejected": -2.375,
      "step": 2640
    },
    {
      "epoch": 5.20117762512267,
      "grad_norm": 10.220288730846836,
      "learning_rate": 2.7558642876219916e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4453125,
      "logps/chosen": -576.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000812530517578125,
      "logps/rejected": -588.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000782012939453125,
      "loss": 0.4264,
      "rewards/accuracies": 0.8650000691413879,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 1.046875,
      "rewards/rejected": -2.484375,
      "step": 2650
    },
    {
      "epoch": 5.220804710500491,
      "grad_norm": 10.499751220520977,
      "learning_rate": 2.7388037451258204e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.40625,
      "logps/chosen": -512.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00084686279296875,
      "logps/rejected": -564.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000843048095703125,
      "loss": 0.4175,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.40625,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -2.578125,
      "step": 2660
    },
    {
      "epoch": 5.240431795878312,
      "grad_norm": 8.557824000197353,
      "learning_rate": 2.721731971625357e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000949859619140625,
      "logps/rejected": -564.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00093841552734375,
      "loss": 0.433,
      "rewards/accuracies": 0.8200000524520874,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 0.91015625,
      "rewards/rejected": -2.25,
      "step": 2670
    },
    {
      "epoch": 5.260058881256134,
      "grad_norm": 9.784128574871701,
      "learning_rate": 2.704649770010696e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000885009765625,
      "logps/rejected": -584.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0008697509765625,
      "loss": 0.4393,
      "rewards/accuracies": 0.7900000214576721,
      "rewards/chosen": -1.515625,
      "rewards/margins": 0.953125,
      "rewards/rejected": -2.46875,
      "step": 2680
    },
    {
      "epoch": 5.279685966633955,
      "grad_norm": 9.663623448103827,
      "learning_rate": 2.6875579436623674e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -490.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000835418701171875,
      "logps/rejected": -556.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000820159912109375,
      "loss": 0.4292,
      "rewards/accuracies": 0.8300000429153442,
      "rewards/chosen": -1.3046875,
      "rewards/margins": 1.015625,
      "rewards/rejected": -2.3125,
      "step": 2690
    },
    {
      "epoch": 5.299313052011776,
      "grad_norm": 11.510812622281577,
      "learning_rate": 2.6704572964135574e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.421875,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.00092315673828125,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.00091552734375,
      "loss": 0.4193,
      "rewards/accuracies": 0.8650000691413879,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -2.640625,
      "step": 2700
    },
    {
      "epoch": 5.299313052011776,
      "eval_logits/chosen": -1.3984375,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -544.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.000885009765625,
      "eval_logps/rejected": -552.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.000873565673828125,
      "eval_loss": 0.5803173780441284,
      "eval_rewards/accuracies": 0.7194030284881592,
      "eval_rewards/chosen": -1.4921875,
      "eval_rewards/margins": 0.65234375,
      "eval_rewards/rejected": -2.140625,
      "eval_runtime": 107.7831,
      "eval_samples_per_second": 18.556,
      "eval_steps_per_second": 0.622,
      "step": 2700
    },
    {
      "epoch": 5.318940137389598,
      "grad_norm": 10.132267872558794,
      "learning_rate": 2.6533486325123004e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4453125,
      "logps/chosen": -482.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000881195068359375,
      "logps/rejected": -532.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000873565673828125,
      "loss": 0.4207,
      "rewards/accuracies": 0.7950000762939453,
      "rewards/chosen": -1.296875,
      "rewards/margins": 0.92578125,
      "rewards/rejected": -2.21875,
      "step": 2710
    },
    {
      "epoch": 5.338567222767419,
      "grad_norm": 8.42201810255113,
      "learning_rate": 2.6362327565836567e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4375,
      "logps/chosen": -494.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000934600830078125,
      "logps/rejected": -560.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00095367431640625,
      "loss": 0.4079,
      "rewards/accuracies": 0.8400000333786011,
      "rewards/chosen": -1.328125,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -2.4375,
      "step": 2720
    },
    {
      "epoch": 5.35819430814524,
      "grad_norm": 12.386861806291925,
      "learning_rate": 2.6191104735918684e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.453125,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000797271728515625,
      "logps/rejected": -580.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000782012939453125,
      "loss": 0.4284,
      "rewards/accuracies": 0.7900000214576721,
      "rewards/chosen": -1.3671875,
      "rewards/margins": 0.90625,
      "rewards/rejected": -2.28125,
      "step": 2730
    },
    {
      "epoch": 5.377821393523062,
      "grad_norm": 9.323175256478242,
      "learning_rate": 2.6019825888025066e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.4140625,
      "logps/chosen": -512.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00086212158203125,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00091552734375,
      "loss": 0.4286,
      "rewards/accuracies": 0.8199999928474426,
      "rewards/chosen": -1.3515625,
      "rewards/margins": 1.0,
      "rewards/rejected": -2.359375,
      "step": 2740
    },
    {
      "epoch": 5.397448478900883,
      "grad_norm": 9.391284546704878,
      "learning_rate": 2.584849907744593e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.421875,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000919342041015625,
      "logps/rejected": -572.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000919342041015625,
      "loss": 0.4222,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.28125,
      "rewards/margins": 1.046875,
      "rewards/rejected": -2.328125,
      "step": 2750
    },
    {
      "epoch": 5.417075564278704,
      "grad_norm": 9.455608857698858,
      "learning_rate": 2.567713236172722e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.4453125,
      "logps/chosen": -504.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000949859619140625,
      "logps/rejected": -556.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00093841552734375,
      "loss": 0.4313,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.3515625,
      "rewards/margins": 1.0546875,
      "rewards/rejected": -2.40625,
      "step": 2760
    },
    {
      "epoch": 5.436702649656526,
      "grad_norm": 9.934725698994821,
      "learning_rate": 2.5505733800291616e-07,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.421875,
      "logps/chosen": -516.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -584.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.4344,
      "rewards/accuracies": 0.7899999618530273,
      "rewards/chosen": -1.359375,
      "rewards/margins": 1.015625,
      "rewards/rejected": -2.375,
      "step": 2770
    },
    {
      "epoch": 5.456329735034347,
      "grad_norm": 9.969441161094824,
      "learning_rate": 2.5334311454059505e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.421875,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00084686279296875,
      "logps/rejected": -572.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000850677490234375,
      "loss": 0.4287,
      "rewards/accuracies": 0.8050000071525574,
      "rewards/chosen": -1.3984375,
      "rewards/margins": 0.9609375,
      "rewards/rejected": -2.359375,
      "step": 2780
    },
    {
      "epoch": 5.4759568204121685,
      "grad_norm": 12.640410652160751,
      "learning_rate": 2.516287338506989e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -556.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000888824462890625,
      "logps/rejected": -572.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000865936279296875,
      "loss": 0.4156,
      "rewards/accuracies": 0.8250001072883606,
      "rewards/chosen": -1.4375,
      "rewards/margins": 1.015625,
      "rewards/rejected": -2.453125,
      "step": 2790
    },
    {
      "epoch": 5.4955839057899905,
      "grad_norm": 9.260418469079971,
      "learning_rate": 2.499142765610122e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -512.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -560.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000911712646484375,
      "loss": 0.419,
      "rewards/accuracies": 0.8300000429153442,
      "rewards/chosen": -1.375,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -2.40625,
      "step": 2800
    },
    {
      "epoch": 5.4955839057899905,
      "eval_logits/chosen": -1.3984375,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -556.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.000949859619140625,
      "eval_logps/rejected": -572.0,
      "eval_logps/rejected_bottom_tokens": -13.875,
      "eval_logps/rejected_top_tokens": -0.000926971435546875,
      "eval_loss": 0.5795263648033142,
      "eval_rewards/accuracies": 0.7194029688835144,
      "eval_rewards/chosen": -1.625,
      "eval_rewards/margins": 0.703125,
      "eval_rewards/rejected": -2.328125,
      "eval_runtime": 107.8021,
      "eval_samples_per_second": 18.553,
      "eval_steps_per_second": 0.622,
      "step": 2800
    },
    {
      "epoch": 5.5152109911678115,
      "grad_norm": 9.426157410126075,
      "learning_rate": 2.481998233029218e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000919342041015625,
      "logps/rejected": -624.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000919342041015625,
      "loss": 0.4261,
      "rewards/accuracies": 0.8149999380111694,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 1.125,
      "rewards/rejected": -2.640625,
      "step": 2810
    },
    {
      "epoch": 5.534838076545633,
      "grad_norm": 7.93562035357333,
      "learning_rate": 2.4648545470762515e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4375,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00093841552734375,
      "logps/rejected": -588.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00090789794921875,
      "loss": 0.4284,
      "rewards/accuracies": 0.7849999666213989,
      "rewards/chosen": -1.5,
      "rewards/margins": 0.8828125,
      "rewards/rejected": -2.375,
      "step": 2820
    },
    {
      "epoch": 5.554465161923455,
      "grad_norm": 10.75663156390518,
      "learning_rate": 2.447712514023378e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.3828125,
      "logps/chosen": -476.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00087738037109375,
      "logps/rejected": -552.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000888824462890625,
      "loss": 0.4362,
      "rewards/accuracies": 0.8199999928474426,
      "rewards/chosen": -1.328125,
      "rewards/margins": 0.98828125,
      "rewards/rejected": -2.3125,
      "step": 2830
    },
    {
      "epoch": 5.574092247301276,
      "grad_norm": 8.242316728901555,
      "learning_rate": 2.4305729400650186e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.3828125,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0008544921875,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00084686279296875,
      "loss": 0.4365,
      "rewards/accuracies": 0.7899999618530273,
      "rewards/chosen": -1.4375,
      "rewards/margins": 0.97265625,
      "rewards/rejected": -2.421875,
      "step": 2840
    },
    {
      "epoch": 5.593719332679097,
      "grad_norm": 9.729387736541266,
      "learning_rate": 2.413436631279941e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.453125,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00077056884765625,
      "logps/rejected": -548.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00074005126953125,
      "loss": 0.4472,
      "rewards/accuracies": 0.7950000166893005,
      "rewards/chosen": -1.375,
      "rewards/margins": 0.92578125,
      "rewards/rejected": -2.296875,
      "step": 2850
    },
    {
      "epoch": 5.613346418056919,
      "grad_norm": 9.579458843278234,
      "learning_rate": 2.3963043935933503e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -460.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00078582763671875,
      "logps/rejected": -540.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00081634521484375,
      "loss": 0.4314,
      "rewards/accuracies": 0.8349999189376831,
      "rewards/chosen": -1.2734375,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -2.359375,
      "step": 2860
    },
    {
      "epoch": 5.63297350343474,
      "grad_norm": 8.387186173885684,
      "learning_rate": 2.3791770327389896e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000873565673828125,
      "logps/rejected": -556.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000858306884765625,
      "loss": 0.4285,
      "rewards/accuracies": 0.8350000381469727,
      "rewards/chosen": -1.328125,
      "rewards/margins": 1.015625,
      "rewards/rejected": -2.34375,
      "step": 2870
    },
    {
      "epoch": 5.652600588812561,
      "grad_norm": 10.151030707964926,
      "learning_rate": 2.3620553542212408e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.390625,
      "logps/chosen": -564.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000926971435546875,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00092315673828125,
      "loss": 0.4249,
      "rewards/accuracies": 0.7949999570846558,
      "rewards/chosen": -1.53125,
      "rewards/margins": 0.9609375,
      "rewards/rejected": -2.5,
      "step": 2880
    },
    {
      "epoch": 5.672227674190383,
      "grad_norm": 8.896938317026654,
      "learning_rate": 2.3449401632772442e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.375,
      "logps/chosen": -510.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00093841552734375,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0009307861328125,
      "loss": 0.4219,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": -1.46875,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -2.515625,
      "step": 2890
    },
    {
      "epoch": 5.691854759568204,
      "grad_norm": 10.861567455177774,
      "learning_rate": 2.3278322648390296e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.421875,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000827789306640625,
      "logps/rejected": -572.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000812530517578125,
      "loss": 0.4267,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 0.9765625,
      "rewards/rejected": -2.484375,
      "step": 2900
    },
    {
      "epoch": 5.691854759568204,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -564.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.0008697509765625,
      "eval_logps/rejected": -576.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.00084686279296875,
      "eval_loss": 0.5779784917831421,
      "eval_rewards/accuracies": 0.713432788848877,
      "eval_rewards/chosen": -1.6875,
      "eval_rewards/margins": 0.68359375,
      "eval_rewards/rejected": -2.375,
      "eval_runtime": 107.7752,
      "eval_samples_per_second": 18.557,
      "eval_steps_per_second": 0.622,
      "step": 2900
    },
    {
      "epoch": 5.711481844946025,
      "grad_norm": 8.927136480999275,
      "learning_rate": 2.3107324634956548e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -564.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0008087158203125,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00083160400390625,
      "loss": 0.4294,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.65625,
      "rewards/margins": 1.03125,
      "rewards/rejected": -2.6875,
      "step": 2910
    },
    {
      "epoch": 5.731108930323847,
      "grad_norm": 10.057562819842476,
      "learning_rate": 2.2936415634553724e-07,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000820159912109375,
      "logps/rejected": -544.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00080108642578125,
      "loss": 0.4347,
      "rewards/accuracies": 0.7899999618530273,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 0.93359375,
      "rewards/rejected": -2.3125,
      "step": 2920
    },
    {
      "epoch": 5.750736015701668,
      "grad_norm": 9.681666890093494,
      "learning_rate": 2.276560368507803e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.0007781982421875,
      "logps/rejected": -624.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000827789306640625,
      "loss": 0.4198,
      "rewards/accuracies": 0.8300000429153442,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 1.0546875,
      "rewards/rejected": -2.53125,
      "step": 2930
    },
    {
      "epoch": 5.770363101079489,
      "grad_norm": 10.195539186974106,
      "learning_rate": 2.2594896819861342e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4453125,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00083160400390625,
      "logps/rejected": -576.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0008392333984375,
      "loss": 0.4316,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 1.109375,
      "rewards/rejected": -2.453125,
      "step": 2940
    },
    {
      "epoch": 5.789990186457311,
      "grad_norm": 9.133704114665807,
      "learning_rate": 2.2424303067293394e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -510.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00090789794921875,
      "logps/rejected": -580.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.4298,
      "rewards/accuracies": 0.8700000643730164,
      "rewards/chosen": -1.4375,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -2.5,
      "step": 2950
    },
    {
      "epoch": 5.809617271835132,
      "grad_norm": 9.738248145069646,
      "learning_rate": 2.22538304504442e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.40625,
      "logps/chosen": -484.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000865936279296875,
      "logps/rejected": -540.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000873565673828125,
      "loss": 0.4294,
      "rewards/accuracies": 0.7900000810623169,
      "rewards/chosen": -1.53125,
      "rewards/margins": 0.8984375,
      "rewards/rejected": -2.421875,
      "step": 2960
    },
    {
      "epoch": 5.8292443572129535,
      "grad_norm": 9.963581731279755,
      "learning_rate": 2.2083486986686737e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.4296875,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00093841552734375,
      "logps/rejected": -624.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00090789794921875,
      "loss": 0.4318,
      "rewards/accuracies": 0.7800000905990601,
      "rewards/chosen": -1.546875,
      "rewards/margins": 0.984375,
      "rewards/rejected": -2.53125,
      "step": 2970
    },
    {
      "epoch": 5.8488714425907755,
      "grad_norm": 9.36785850903005,
      "learning_rate": 2.191328068731987e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.421875,
      "logps/chosen": -510.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00107574462890625,
      "logps/rejected": -568.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00109100341796875,
      "loss": 0.4178,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 1.0078125,
      "rewards/rejected": -2.53125,
      "step": 2980
    },
    {
      "epoch": 5.868498527968597,
      "grad_norm": 9.456033086378868,
      "learning_rate": 2.1743219557191583e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00086212158203125,
      "loss": 0.4245,
      "rewards/accuracies": 0.8349999189376831,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -2.578125,
      "step": 2990
    },
    {
      "epoch": 5.888125613346418,
      "grad_norm": 9.12907293722466,
      "learning_rate": 2.1573311594322527e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.390625,
      "logps/chosen": -516.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0010986328125,
      "logps/rejected": -576.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00109100341796875,
      "loss": 0.402,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4375,
      "rewards/margins": 1.046875,
      "rewards/rejected": -2.484375,
      "step": 3000
    },
    {
      "epoch": 5.888125613346418,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4453125,
      "eval_logps/chosen": -560.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.000919342041015625,
      "eval_logps/rejected": -572.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.000888824462890625,
      "eval_loss": 0.5827575922012329,
      "eval_rewards/accuracies": 0.7253730893135071,
      "eval_rewards/chosen": -1.6484375,
      "eval_rewards/margins": 0.7109375,
      "eval_rewards/rejected": -2.359375,
      "eval_runtime": 107.8231,
      "eval_samples_per_second": 18.549,
      "eval_steps_per_second": 0.621,
      "step": 3000
    },
    {
      "epoch": 5.90775269872424,
      "grad_norm": 12.532499318616896,
      "learning_rate": 2.1403564789529833e-07,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.4375,
      "logps/chosen": -512.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000873565673828125,
      "logps/rejected": -564.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0008544921875,
      "loss": 0.4136,
      "rewards/accuracies": 0.8199999928474426,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 1.0,
      "rewards/rejected": -2.4375,
      "step": 3010
    },
    {
      "epoch": 5.927379784102061,
      "grad_norm": 10.314331752317209,
      "learning_rate": 2.123398712605134e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4375,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000934600830078125,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00092315673828125,
      "loss": 0.4162,
      "rewards/accuracies": 0.8350000381469727,
      "rewards/chosen": -1.46875,
      "rewards/margins": 1.03125,
      "rewards/rejected": -2.5,
      "step": 3020
    },
    {
      "epoch": 5.947006869479882,
      "grad_norm": 8.665989364072294,
      "learning_rate": 2.1064586579170121e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -482.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0008697509765625,
      "logps/rejected": -548.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00086212158203125,
      "loss": 0.4144,
      "rewards/accuracies": 0.8050000071525574,
      "rewards/chosen": -1.4609375,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -2.484375,
      "step": 3030
    },
    {
      "epoch": 5.966633954857704,
      "grad_norm": 9.103364254357956,
      "learning_rate": 2.0895371115839412e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -528.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00079345703125,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000797271728515625,
      "loss": 0.4207,
      "rewards/accuracies": 0.8449999094009399,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -2.359375,
      "step": 3040
    },
    {
      "epoch": 5.986261040235525,
      "grad_norm": 8.380145453097748,
      "learning_rate": 2.0726348694307914e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.453125,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000782012939453125,
      "logps/rejected": -596.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00078582763671875,
      "loss": 0.4266,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 1.109375,
      "rewards/rejected": -2.546875,
      "step": 3050
    },
    {
      "epoch": 6.005888125613346,
      "grad_norm": 10.887214254749555,
      "learning_rate": 2.0557527263745523e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00087738037109375,
      "logps/rejected": -544.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000858306884765625,
      "loss": 0.4071,
      "rewards/accuracies": 0.8100000619888306,
      "rewards/chosen": -1.3359375,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -2.375,
      "step": 3060
    },
    {
      "epoch": 6.025515210991168,
      "grad_norm": 9.041321401770846,
      "learning_rate": 2.0388914763869478e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.453125,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000820159912109375,
      "logps/rejected": -576.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000812530517578125,
      "loss": 0.396,
      "rewards/accuracies": 0.809999942779541,
      "rewards/chosen": -1.453125,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -2.515625,
      "step": 3070
    },
    {
      "epoch": 6.045142296368989,
      "grad_norm": 8.810241188671984,
      "learning_rate": 2.0220519124570944e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.4296875,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0009307861328125,
      "logps/rejected": -556.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000888824462890625,
      "loss": 0.3898,
      "rewards/accuracies": 0.8449999690055847,
      "rewards/chosen": -1.4609375,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -2.53125,
      "step": 3080
    },
    {
      "epoch": 6.06476938174681,
      "grad_norm": 9.444456329600422,
      "learning_rate": 2.0052348265542086e-07,
      "logits/chosen": -1.4296875,
      "logits/rejected": -1.453125,
      "logps/chosen": -512.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000835418701171875,
      "logps/rejected": -568.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000820159912109375,
      "loss": 0.3843,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.3203125,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -2.5625,
      "step": 3090
    },
    {
      "epoch": 6.084396467124632,
      "grad_norm": 9.592468437934086,
      "learning_rate": 1.9884410095903584e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -486.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000888824462890625,
      "logps/rejected": -564.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.00087738037109375,
      "loss": 0.3656,
      "rewards/accuracies": 0.8449999690055847,
      "rewards/chosen": -1.5,
      "rewards/margins": 1.25,
      "rewards/rejected": -2.75,
      "step": 3100
    },
    {
      "epoch": 6.084396467124632,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -564.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.00089263916015625,
      "eval_logps/rejected": -580.0,
      "eval_logps/rejected_bottom_tokens": -13.875,
      "eval_logps/rejected_top_tokens": -0.0008697509765625,
      "eval_loss": 0.5844067335128784,
      "eval_rewards/accuracies": 0.7014926075935364,
      "eval_rewards/chosen": -1.6875,
      "eval_rewards/margins": 0.72265625,
      "eval_rewards/rejected": -2.40625,
      "eval_runtime": 107.8624,
      "eval_samples_per_second": 18.542,
      "eval_steps_per_second": 0.621,
      "step": 3100
    },
    {
      "epoch": 6.104023552502453,
      "grad_norm": 11.239398087240554,
      "learning_rate": 1.971671251383268e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00083160400390625,
      "logps/rejected": -572.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000820159912109375,
      "loss": 0.4034,
      "rewards/accuracies": 0.8050000071525574,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 0.8984375,
      "rewards/rejected": -2.421875,
      "step": 3110
    },
    {
      "epoch": 6.123650637880274,
      "grad_norm": 11.234299981879975,
      "learning_rate": 1.9549263406191703e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.40625,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00087738037109375,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000835418701171875,
      "loss": 0.4049,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 1.140625,
      "rewards/rejected": -2.6875,
      "step": 3120
    },
    {
      "epoch": 6.143277723258096,
      "grad_norm": 11.498877753164736,
      "learning_rate": 1.9382070648157184e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -504.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.000911712646484375,
      "logps/rejected": -564.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.0009307861328125,
      "loss": 0.3899,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.3671875,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -2.5625,
      "step": 3130
    },
    {
      "epoch": 6.1629048086359175,
      "grad_norm": 10.004673273559828,
      "learning_rate": 1.9215142102849448e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00112152099609375,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.001129150390625,
      "loss": 0.4067,
      "rewards/accuracies": 0.7499999403953552,
      "rewards/chosen": -1.59375,
      "rewards/margins": 0.9375,
      "rewards/rejected": -2.53125,
      "step": 3140
    },
    {
      "epoch": 6.182531894013739,
      "grad_norm": 9.220472070105401,
      "learning_rate": 1.904848562096283e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4375,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0008697509765625,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000865936279296875,
      "loss": 0.3962,
      "rewards/accuracies": 0.8050000071525574,
      "rewards/chosen": -1.53125,
      "rewards/margins": 1.0625,
      "rewards/rejected": -2.59375,
      "step": 3150
    },
    {
      "epoch": 6.202158979391561,
      "grad_norm": 10.124298050031104,
      "learning_rate": 1.8882109040396455e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -496.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.00087738037109375,
      "logps/rejected": -588.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000873565673828125,
      "loss": 0.3954,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.40625,
      "rewards/margins": 1.21875,
      "rewards/rejected": -2.625,
      "step": 3160
    },
    {
      "epoch": 6.221786064769382,
      "grad_norm": 11.485253390722129,
      "learning_rate": 1.8716020185885597e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -508.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0009307861328125,
      "logps/rejected": -556.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000904083251953125,
      "loss": 0.3863,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4375,
      "rewards/margins": 1.046875,
      "rewards/rejected": -2.484375,
      "step": 3170
    },
    {
      "epoch": 6.241413150147203,
      "grad_norm": 9.627394491057656,
      "learning_rate": 1.8550226868633717e-07,
      "logits/chosen": -1.4296875,
      "logits/rejected": -1.421875,
      "logps/chosen": -470.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00086212158203125,
      "logps/rejected": -580.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.3882,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": -1.5,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -2.640625,
      "step": 3180
    },
    {
      "epoch": 6.261040235525025,
      "grad_norm": 9.841950410892725,
      "learning_rate": 1.8384736885945059e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -508.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0007781982421875,
      "logps/rejected": -588.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00077056884765625,
      "loss": 0.3849,
      "rewards/accuracies": 0.8750001192092896,
      "rewards/chosen": -1.484375,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -2.671875,
      "step": 3190
    },
    {
      "epoch": 6.280667320902846,
      "grad_norm": 11.628550681334719,
      "learning_rate": 1.8219558020857978e-07,
      "logits/chosen": -1.453125,
      "logits/rejected": -1.4375,
      "logps/chosen": -492.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000942230224609375,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00096893310546875,
      "loss": 0.3971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -2.8125,
      "step": 3200
    },
    {
      "epoch": 6.280667320902846,
      "eval_logits/chosen": -1.4140625,
      "eval_logits/rejected": -1.4453125,
      "eval_logps/chosen": -556.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.0009002685546875,
      "eval_logps/rejected": -572.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.000865936279296875,
      "eval_loss": 0.5873156785964966,
      "eval_rewards/accuracies": 0.707462728023529,
      "eval_rewards/chosen": -1.609375,
      "eval_rewards/margins": 0.71484375,
      "eval_rewards/rejected": -2.328125,
      "eval_runtime": 107.9708,
      "eval_samples_per_second": 18.524,
      "eval_steps_per_second": 0.621,
      "step": 3200
    },
    {
      "epoch": 6.300294406280667,
      "grad_norm": 9.556898634263252,
      "learning_rate": 1.8054698041778877e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0009765625,
      "logps/rejected": -596.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00096893310546875,
      "loss": 0.3902,
      "rewards/accuracies": 0.7849999666213989,
      "rewards/chosen": -1.609375,
      "rewards/margins": 1.0,
      "rewards/rejected": -2.609375,
      "step": 3210
    },
    {
      "epoch": 6.319921491658489,
      "grad_norm": 16.752069032509507,
      "learning_rate": 1.7890164702116866e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.4375,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000885009765625,
      "logps/rejected": -576.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0008544921875,
      "loss": 0.4008,
      "rewards/accuracies": 0.815000057220459,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 0.99609375,
      "rewards/rejected": -2.578125,
      "step": 3220
    },
    {
      "epoch": 6.33954857703631,
      "grad_norm": 11.997211906892314,
      "learning_rate": 1.772596573991911e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.453125,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0009002685546875,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.387,
      "rewards/accuracies": 0.8449999690055847,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -2.671875,
      "step": 3230
    },
    {
      "epoch": 6.359175662414131,
      "grad_norm": 9.293641108361177,
      "learning_rate": 1.7562108877506917e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -516.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000934600830078125,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000957489013671875,
      "loss": 0.3954,
      "rewards/accuracies": 0.8299999237060547,
      "rewards/chosen": -1.578125,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -2.65625,
      "step": 3240
    },
    {
      "epoch": 6.378802747791953,
      "grad_norm": 11.246848893505113,
      "learning_rate": 1.7398601821112552e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4453125,
      "logps/chosen": -588.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00089263916015625,
      "logps/rejected": -624.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000888824462890625,
      "loss": 0.3935,
      "rewards/accuracies": 0.8950001001358032,
      "rewards/chosen": -1.515625,
      "rewards/margins": 1.34375,
      "rewards/rejected": -2.859375,
      "step": 3250
    },
    {
      "epoch": 6.398429833169774,
      "grad_norm": 11.258833049062101,
      "learning_rate": 1.7235452260516803e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -512.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000942230224609375,
      "logps/rejected": -564.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000919342041015625,
      "loss": 0.4025,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": -1.5,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -2.625,
      "step": 3260
    },
    {
      "epoch": 6.418056918547595,
      "grad_norm": 8.92258190248652,
      "learning_rate": 1.7072667868687346e-07,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -528.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000881195068359375,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00087738037109375,
      "loss": 0.3781,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": -1.546875,
      "rewards/margins": 1.09375,
      "rewards/rejected": -2.640625,
      "step": 3270
    },
    {
      "epoch": 6.437684003925417,
      "grad_norm": 11.88168151340057,
      "learning_rate": 1.6910256301417854e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -484.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00098419189453125,
      "logps/rejected": -540.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000965118408203125,
      "loss": 0.3906,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 1.140625,
      "rewards/rejected": -2.578125,
      "step": 3280
    },
    {
      "epoch": 6.457311089303238,
      "grad_norm": 11.633081473254599,
      "learning_rate": 1.674822519696798e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.3984375,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -628.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000896453857421875,
      "loss": 0.3991,
      "rewards/accuracies": 0.8050000071525574,
      "rewards/chosen": -1.4921875,
      "rewards/margins": 1.0,
      "rewards/rejected": -2.5,
      "step": 3290
    },
    {
      "epoch": 6.4769381746810595,
      "grad_norm": 11.362089125815567,
      "learning_rate": 1.6586582175704088e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4375,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000835418701171875,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00084686279296875,
      "loss": 0.3923,
      "rewards/accuracies": 0.85999995470047,
      "rewards/chosen": -1.4140625,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -2.625,
      "step": 3300
    },
    {
      "epoch": 6.4769381746810595,
      "eval_logits/chosen": -1.4140625,
      "eval_logits/rejected": -1.4453125,
      "eval_logps/chosen": -564.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.000896453857421875,
      "eval_logps/rejected": -580.0,
      "eval_logps/rejected_bottom_tokens": -13.875,
      "eval_logps/rejected_top_tokens": -0.000865936279296875,
      "eval_loss": 0.590624988079071,
      "eval_rewards/accuracies": 0.707462728023529,
      "eval_rewards/chosen": -1.6875,
      "eval_rewards/margins": 0.71875,
      "eval_rewards/rejected": -2.40625,
      "eval_runtime": 108.0588,
      "eval_samples_per_second": 18.508,
      "eval_steps_per_second": 0.62,
      "step": 3300
    },
    {
      "epoch": 6.4965652600588815,
      "grad_norm": 12.198850746961845,
      "learning_rate": 1.6425334839740912e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4375,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000835418701171875,
      "logps/rejected": -628.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000865936279296875,
      "loss": 0.395,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.46875,
      "rewards/margins": 1.3125,
      "rewards/rejected": -2.78125,
      "step": 3310
    },
    {
      "epoch": 6.516192345436703,
      "grad_norm": 11.826666381899823,
      "learning_rate": 1.6264490772583984e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -576.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00081634521484375,
      "logps/rejected": -636.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000820159912109375,
      "loss": 0.3971,
      "rewards/accuracies": 0.8449999690055847,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -2.71875,
      "step": 3320
    },
    {
      "epoch": 6.535819430814524,
      "grad_norm": 9.92386217929032,
      "learning_rate": 1.6104057538772974e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.390625,
      "logps/chosen": -496.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00090789794921875,
      "logps/rejected": -580.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000926971435546875,
      "loss": 0.3988,
      "rewards/accuracies": 0.8300000429153442,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -2.796875,
      "step": 3330
    },
    {
      "epoch": 6.555446516192346,
      "grad_norm": 10.541347410096726,
      "learning_rate": 1.594404268352599e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000843048095703125,
      "loss": 0.409,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": -1.609375,
      "rewards/margins": 0.93359375,
      "rewards/rejected": -2.546875,
      "step": 3340
    },
    {
      "epoch": 6.575073601570167,
      "grad_norm": 10.351560745409461,
      "learning_rate": 1.5784453732384651e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0008544921875,
      "logps/rejected": -588.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000888824462890625,
      "loss": 0.3885,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 1.1875,
      "rewards/rejected": -2.703125,
      "step": 3350
    },
    {
      "epoch": 6.594700686947988,
      "grad_norm": 9.362040923183626,
      "learning_rate": 1.5625298190860226e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3828125,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000885009765625,
      "logps/rejected": -624.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000865936279296875,
      "loss": 0.387,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 1.09375,
      "rewards/rejected": -2.65625,
      "step": 3360
    },
    {
      "epoch": 6.61432777232581,
      "grad_norm": 11.80049485106858,
      "learning_rate": 1.5466583544080585e-07,
      "logits/chosen": -1.4296875,
      "logits/rejected": -1.4453125,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000850677490234375,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000835418701171875,
      "loss": 0.3889,
      "rewards/accuracies": 0.9049999117851257,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 1.1875,
      "rewards/rejected": -2.625,
      "step": 3370
    },
    {
      "epoch": 6.633954857703631,
      "grad_norm": 10.067392531938895,
      "learning_rate": 1.5308317256438203e-07,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4375,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000888824462890625,
      "logps/rejected": -576.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00089263916015625,
      "loss": 0.3836,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.40625,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -2.640625,
      "step": 3380
    },
    {
      "epoch": 6.653581943081452,
      "grad_norm": 8.6323831027187,
      "learning_rate": 1.5150506771239114e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -482.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00086212158203125,
      "logps/rejected": -580.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0008697509765625,
      "loss": 0.3968,
      "rewards/accuracies": 0.8350000381469727,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 1.0625,
      "rewards/rejected": -2.546875,
      "step": 3390
    },
    {
      "epoch": 6.673209028459274,
      "grad_norm": 11.24553545655022,
      "learning_rate": 1.4993159510352835e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4453125,
      "logps/chosen": -528.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00081634521484375,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0008087158203125,
      "loss": 0.4011,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.375,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -2.625,
      "step": 3400
    },
    {
      "epoch": 6.673209028459274,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -564.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.000873565673828125,
      "eval_logps/rejected": -584.0,
      "eval_logps/rejected_bottom_tokens": -13.875,
      "eval_logps/rejected_top_tokens": -0.000835418701171875,
      "eval_loss": 0.5848119854927063,
      "eval_rewards/accuracies": 0.7253730893135071,
      "eval_rewards/chosen": -1.7109375,
      "eval_rewards/margins": 0.734375,
      "eval_rewards/rejected": -2.4375,
      "eval_runtime": 108.0607,
      "eval_samples_per_second": 18.508,
      "eval_steps_per_second": 0.62,
      "step": 3400
    },
    {
      "epoch": 6.692836113837095,
      "grad_norm": 11.39250478443067,
      "learning_rate": 1.4836282873863317e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.4296875,
      "logps/chosen": -548.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00079345703125,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0007781982421875,
      "loss": 0.3918,
      "rewards/accuracies": 0.8400000333786011,
      "rewards/chosen": -1.625,
      "rewards/margins": 1.0625,
      "rewards/rejected": -2.6875,
      "step": 3410
    },
    {
      "epoch": 6.712463199214916,
      "grad_norm": 10.723016747400582,
      "learning_rate": 1.4679884239720927e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.4140625,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00091552734375,
      "logps/rejected": -596.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000873565673828125,
      "loss": 0.3919,
      "rewards/accuracies": 0.85999995470047,
      "rewards/chosen": -1.5,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -2.625,
      "step": 3420
    },
    {
      "epoch": 6.732090284592738,
      "grad_norm": 10.42018472664649,
      "learning_rate": 1.4523970963395448e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.40625,
      "logps/chosen": -528.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -568.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00086212158203125,
      "loss": 0.39,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4921875,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -2.53125,
      "step": 3430
    },
    {
      "epoch": 6.751717369970559,
      "grad_norm": 10.208772210201792,
      "learning_rate": 1.436855037753016e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.40625,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000865936279296875,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000858306884765625,
      "loss": 0.3841,
      "rewards/accuracies": 0.8799999356269836,
      "rewards/chosen": -1.546875,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -2.65625,
      "step": 3440
    },
    {
      "epoch": 6.77134445534838,
      "grad_norm": 12.551474540069226,
      "learning_rate": 1.4213629791596948e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -548.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000946044921875,
      "logps/rejected": -648.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00102996826171875,
      "loss": 0.3924,
      "rewards/accuracies": 0.85999995470047,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -2.59375,
      "step": 3450
    },
    {
      "epoch": 6.790971540726202,
      "grad_norm": 9.157599801693973,
      "learning_rate": 1.405921649155262e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.40625,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000774383544921875,
      "logps/rejected": -644.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000804901123046875,
      "loss": 0.3838,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 1.1484375,
      "rewards/rejected": -2.65625,
      "step": 3460
    },
    {
      "epoch": 6.8105986261040234,
      "grad_norm": 10.960666918595264,
      "learning_rate": 1.390531773949614e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.40625,
      "logps/chosen": -548.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000919342041015625,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.000896453857421875,
      "loss": 0.3907,
      "rewards/accuracies": 0.8749998807907104,
      "rewards/chosen": -1.484375,
      "rewards/margins": 1.234375,
      "rewards/rejected": -2.71875,
      "step": 3470
    },
    {
      "epoch": 6.8302257114818445,
      "grad_norm": 9.769484255696572,
      "learning_rate": 1.3751940773327192e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.421875,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000823974609375,
      "logps/rejected": -640.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00083160400390625,
      "loss": 0.3914,
      "rewards/accuracies": 0.8549998998641968,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -2.734375,
      "step": 3480
    },
    {
      "epoch": 6.8498527968596665,
      "grad_norm": 11.696207657866864,
      "learning_rate": 1.3599092806405675e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.40625,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000873565673828125,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00086212158203125,
      "loss": 0.3816,
      "rewards/accuracies": 0.8449999690055847,
      "rewards/chosen": -1.6875,
      "rewards/margins": 1.09375,
      "rewards/rejected": -2.78125,
      "step": 3490
    },
    {
      "epoch": 6.869479882237488,
      "grad_norm": 11.278674812229928,
      "learning_rate": 1.3446781027212562e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.421875,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0010223388671875,
      "logps/rejected": -584.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0010223388671875,
      "loss": 0.3838,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -2.703125,
      "step": 3500
    },
    {
      "epoch": 6.869479882237488,
      "eval_logits/chosen": -1.3984375,
      "eval_logits/rejected": -1.4296875,
      "eval_logps/chosen": -568.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.00087738037109375,
      "eval_logps/rejected": -584.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.00083160400390625,
      "eval_loss": 0.5897497534751892,
      "eval_rewards/accuracies": 0.7164179086685181,
      "eval_rewards/chosen": -1.75,
      "eval_rewards/margins": 0.73046875,
      "eval_rewards/rejected": -2.484375,
      "eval_runtime": 107.9889,
      "eval_samples_per_second": 18.52,
      "eval_steps_per_second": 0.62,
      "step": 3500
    },
    {
      "epoch": 6.889106967615309,
      "grad_norm": 12.55807014514054,
      "learning_rate": 1.3295012599011729e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.390625,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000804901123046875,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.00081634521484375,
      "loss": 0.4028,
      "rewards/accuracies": 0.7950000166893005,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 0.8984375,
      "rewards/rejected": -2.609375,
      "step": 3510
    },
    {
      "epoch": 6.908734052993131,
      "grad_norm": 13.054968647766866,
      "learning_rate": 1.3143794659513152e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4375,
      "logps/chosen": -588.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00080108642578125,
      "logps/rejected": -660.0,
      "logps/rejected_bottom_tokens": -14.0,
      "logps/rejected_top_tokens": -0.000774383544921875,
      "loss": 0.393,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": -1.640625,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -2.84375,
      "step": 3520
    },
    {
      "epoch": 6.928361138370952,
      "grad_norm": 12.628737963360468,
      "learning_rate": 1.299313432053713e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.0009307861328125,
      "logps/rejected": -576.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000957489013671875,
      "loss": 0.3908,
      "rewards/accuracies": 0.8449999690055847,
      "rewards/chosen": -1.4375,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -2.53125,
      "step": 3530
    },
    {
      "epoch": 6.947988223748773,
      "grad_norm": 11.20799763262095,
      "learning_rate": 1.2843038667679903e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00081634521484375,
      "logps/rejected": -584.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00079345703125,
      "loss": 0.4002,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4140625,
      "rewards/margins": 1.15625,
      "rewards/rejected": -2.578125,
      "step": 3540
    },
    {
      "epoch": 6.967615309126595,
      "grad_norm": 11.678629068520095,
      "learning_rate": 1.2693514759980345e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.4375,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00093841552734375,
      "logps/rejected": -540.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000934600830078125,
      "loss": 0.3873,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 1.0625,
      "rewards/rejected": -2.59375,
      "step": 3550
    },
    {
      "epoch": 6.987242394504416,
      "grad_norm": 11.651443048479592,
      "learning_rate": 1.2544569629587994e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3828125,
      "logps/chosen": -506.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000919342041015625,
      "logps/rejected": -580.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000934600830078125,
      "loss": 0.3999,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": -1.4140625,
      "rewards/margins": 1.171875,
      "rewards/rejected": -2.59375,
      "step": 3560
    },
    {
      "epoch": 7.006869479882237,
      "grad_norm": 9.376535561651522,
      "learning_rate": 1.2396210281432375e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.390625,
      "logps/chosen": -516.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0009002685546875,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00091552734375,
      "loss": 0.3612,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": -1.4140625,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -2.765625,
      "step": 3570
    },
    {
      "epoch": 7.026496565260059,
      "grad_norm": 13.057258643450153,
      "learning_rate": 1.2248443692893462e-07,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00083160400390625,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000797271728515625,
      "loss": 0.3734,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -2.609375,
      "step": 3580
    },
    {
      "epoch": 7.04612365063788,
      "grad_norm": 10.72261951423499,
      "learning_rate": 1.210127681347364e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.3828125,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0008544921875,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000888824462890625,
      "loss": 0.3647,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -2.671875,
      "step": 3590
    },
    {
      "epoch": 7.065750736015701,
      "grad_norm": 9.306580817942635,
      "learning_rate": 1.1954716564470771e-07,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.40625,
      "logps/chosen": -488.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000885009765625,
      "logps/rejected": -556.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000896453857421875,
      "loss": 0.3762,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -2.625,
      "step": 3600
    },
    {
      "epoch": 7.065750736015701,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -572.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.000865936279296875,
      "eval_logps/rejected": -592.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.000823974609375,
      "eval_loss": 0.5909960865974426,
      "eval_rewards/accuracies": 0.7134329080581665,
      "eval_rewards/chosen": -1.78125,
      "eval_rewards/margins": 0.7421875,
      "eval_rewards/rejected": -2.53125,
      "eval_runtime": 108.0788,
      "eval_samples_per_second": 18.505,
      "eval_steps_per_second": 0.62,
      "step": 3600
    },
    {
      "epoch": 7.085377821393523,
      "grad_norm": 11.891661500010164,
      "learning_rate": 1.1808769838652755e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.40625,
      "logps/chosen": -592.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000812530517578125,
      "loss": 0.3741,
      "rewards/accuracies": 0.8449999690055847,
      "rewards/chosen": -1.65625,
      "rewards/margins": 1.046875,
      "rewards/rejected": -2.703125,
      "step": 3610
    },
    {
      "epoch": 7.105004906771344,
      "grad_norm": 10.69499765785664,
      "learning_rate": 1.1663443499933301e-07,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.000919342041015625,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.000919342041015625,
      "loss": 0.3561,
      "rewards/accuracies": 0.9199999570846558,
      "rewards/chosen": -1.53125,
      "rewards/margins": 1.296875,
      "rewards/rejected": -2.828125,
      "step": 3620
    },
    {
      "epoch": 7.124631992149165,
      "grad_norm": 11.278287242230846,
      "learning_rate": 1.1518744383049187e-07,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.40625,
      "logps/chosen": -580.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000812530517578125,
      "logps/rejected": -628.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00080108642578125,
      "loss": 0.3571,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 1.25,
      "rewards/rejected": -2.90625,
      "step": 3630
    },
    {
      "epoch": 7.144259077526987,
      "grad_norm": 10.714223580426642,
      "learning_rate": 1.1374679293238731e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.453125,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000850677490234375,
      "logps/rejected": -640.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000850677490234375,
      "loss": 0.3737,
      "rewards/accuracies": 0.8549998998641968,
      "rewards/chosen": -1.578125,
      "rewards/margins": 1.34375,
      "rewards/rejected": -2.921875,
      "step": 3640
    },
    {
      "epoch": 7.1638861629048085,
      "grad_norm": 13.726506022363308,
      "learning_rate": 1.1231255005921845e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.453125,
      "logps/chosen": -556.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000911712646484375,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0008697509765625,
      "loss": 0.3612,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 1.109375,
      "rewards/rejected": -2.625,
      "step": 3650
    },
    {
      "epoch": 7.18351324828263,
      "grad_norm": 13.79859784191235,
      "learning_rate": 1.1088478266381257e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -478.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000946044921875,
      "logps/rejected": -596.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.0009613037109375,
      "loss": 0.3681,
      "rewards/accuracies": 0.9050000309944153,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 1.328125,
      "rewards/rejected": -2.78125,
      "step": 3660
    },
    {
      "epoch": 7.203140333660452,
      "grad_norm": 12.54064257210143,
      "learning_rate": 1.0946355789445407e-07,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000843048095703125,
      "logps/rejected": -628.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00083160400390625,
      "loss": 0.3771,
      "rewards/accuracies": 0.8199998736381531,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -2.875,
      "step": 3670
    },
    {
      "epoch": 7.222767419038273,
      "grad_norm": 10.115831687928802,
      "learning_rate": 1.0804894259172579e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.421875,
      "logps/chosen": -528.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00086212158203125,
      "logps/rejected": -572.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000823974609375,
      "loss": 0.3621,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.4921875,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -2.703125,
      "step": 3680
    },
    {
      "epoch": 7.242394504416094,
      "grad_norm": 11.603121001252209,
      "learning_rate": 1.0664100328536523e-07,
      "logits/chosen": -1.4296875,
      "logits/rejected": -1.4609375,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000888824462890625,
      "loss": 0.3819,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.453125,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -2.703125,
      "step": 3690
    },
    {
      "epoch": 7.262021589793916,
      "grad_norm": 9.168794884896538,
      "learning_rate": 1.0523980619113654e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000888824462890625,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000896453857421875,
      "loss": 0.3591,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 1.234375,
      "rewards/rejected": -2.828125,
      "step": 3700
    },
    {
      "epoch": 7.262021589793916,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -572.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.00090789794921875,
      "eval_logps/rejected": -592.0,
      "eval_logps/rejected_bottom_tokens": -13.875,
      "eval_logps/rejected_top_tokens": -0.0008697509765625,
      "eval_loss": 0.5895251631736755,
      "eval_rewards/accuracies": 0.707462728023529,
      "eval_rewards/chosen": -1.78125,
      "eval_rewards/margins": 0.7578125,
      "eval_rewards/rejected": -2.53125,
      "eval_runtime": 108.0245,
      "eval_samples_per_second": 18.514,
      "eval_steps_per_second": 0.62,
      "step": 3700
    },
    {
      "epoch": 7.281648675171737,
      "grad_norm": 10.850056329380424,
      "learning_rate": 1.0384541720771522e-07,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0008697509765625,
      "logps/rejected": -640.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000881195068359375,
      "loss": 0.3514,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -2.84375,
      "step": 3710
    },
    {
      "epoch": 7.301275760549558,
      "grad_norm": 11.485411143489458,
      "learning_rate": 1.0245790191359007e-07,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4453125,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.00087738037109375,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000858306884765625,
      "loss": 0.3556,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.578125,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -2.828125,
      "step": 3720
    },
    {
      "epoch": 7.32090284592738,
      "grad_norm": 11.82455858645229,
      "learning_rate": 1.0107732556397791e-07,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.421875,
      "logps/chosen": -510.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.00103759765625,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00103759765625,
      "loss": 0.3764,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 1.1875,
      "rewards/rejected": -2.78125,
      "step": 3730
    },
    {
      "epoch": 7.340529931305201,
      "grad_norm": 11.475611182028112,
      "learning_rate": 9.970375308775559e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.00091552734375,
      "logps/rejected": -636.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000904083251953125,
      "loss": 0.3561,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": -1.515625,
      "rewards/margins": 1.21875,
      "rewards/rejected": -2.734375,
      "step": 3740
    },
    {
      "epoch": 7.360157016683022,
      "grad_norm": 12.197757170153134,
      "learning_rate": 9.833724908440561e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000812530517578125,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000827789306640625,
      "loss": 0.3582,
      "rewards/accuracies": 0.8850000500679016,
      "rewards/chosen": -1.4453125,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -2.75,
      "step": 3750
    },
    {
      "epoch": 7.379784102060844,
      "grad_norm": 10.738153861615361,
      "learning_rate": 9.697787782097836e-08,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.3828125,
      "logps/chosen": -548.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0009002685546875,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.3722,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.4765625,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -2.796875,
      "step": 3760
    },
    {
      "epoch": 7.399411187438665,
      "grad_norm": 21.827539496326647,
      "learning_rate": 9.562570322906952e-08,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.453125,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000865936279296875,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0008392333984375,
      "loss": 0.3814,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 1.0390625,
      "rewards/rejected": -2.6875,
      "step": 3770
    },
    {
      "epoch": 7.419038272816486,
      "grad_norm": 10.133341733647612,
      "learning_rate": 9.428078890181362e-08,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.453125,
      "logps/chosen": -620.0,
      "logps/chosen_bottom_tokens": -14.1875,
      "logps/chosen_top_tokens": -0.00078582763671875,
      "logps/rejected": -672.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000762939453125,
      "loss": 0.3636,
      "rewards/accuracies": 0.8650000691413879,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 1.21875,
      "rewards/rejected": -2.953125,
      "step": 3780
    },
    {
      "epoch": 7.438665358194308,
      "grad_norm": 11.414588203095605,
      "learning_rate": 9.294319809089262e-08,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.40625,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00092315673828125,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00090789794921875,
      "loss": 0.3786,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": -1.546875,
      "rewards/margins": 1.25,
      "rewards/rejected": -2.796875,
      "step": 3790
    },
    {
      "epoch": 7.458292443572129,
      "grad_norm": 11.04997051979466,
      "learning_rate": 9.16129937035619e-08,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0009002685546875,
      "logps/rejected": -588.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0009307861328125,
      "loss": 0.3713,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.5,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -2.734375,
      "step": 3800
    },
    {
      "epoch": 7.458292443572129,
      "eval_logits/chosen": -1.3984375,
      "eval_logits/rejected": -1.4296875,
      "eval_logps/chosen": -572.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.00089263916015625,
      "eval_logps/rejected": -592.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.000858306884765625,
      "eval_loss": 0.5956262350082397,
      "eval_rewards/accuracies": 0.7164179086685181,
      "eval_rewards/chosen": -1.7734375,
      "eval_rewards/margins": 0.75,
      "eval_rewards/rejected": -2.53125,
      "eval_runtime": 108.1243,
      "eval_samples_per_second": 18.497,
      "eval_steps_per_second": 0.62,
      "step": 3800
    },
    {
      "epoch": 7.4779195289499505,
      "grad_norm": 13.067662208055344,
      "learning_rate": 9.029023829969102e-08,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.4140625,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -624.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0008544921875,
      "loss": 0.3714,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -2.84375,
      "step": 3810
    },
    {
      "epoch": 7.4975466143277725,
      "grad_norm": 12.33759924091816,
      "learning_rate": 8.897499408882206e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000850677490234375,
      "logps/rejected": -588.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000820159912109375,
      "loss": 0.3629,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 1.1328125,
      "rewards/rejected": -2.71875,
      "step": 3820
    },
    {
      "epoch": 7.517173699705594,
      "grad_norm": 13.798167173719602,
      "learning_rate": 8.766732292724377e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00081634521484375,
      "logps/rejected": -636.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0008087158203125,
      "loss": 0.3653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6875,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -2.9375,
      "step": 3830
    },
    {
      "epoch": 7.536800785083415,
      "grad_norm": 12.333957733743242,
      "learning_rate": 8.636728631508198e-08,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.40625,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000919342041015625,
      "logps/rejected": -596.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000896453857421875,
      "loss": 0.3772,
      "rewards/accuracies": 0.8200000524520874,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 1.1484375,
      "rewards/rejected": -2.890625,
      "step": 3840
    },
    {
      "epoch": 7.556427870461237,
      "grad_norm": 11.995779984555064,
      "learning_rate": 8.507494539340804e-08,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.3984375,
      "logps/chosen": -572.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000888824462890625,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0008544921875,
      "loss": 0.3747,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.5625,
      "rewards/margins": 1.234375,
      "rewards/rejected": -2.796875,
      "step": 3850
    },
    {
      "epoch": 7.576054955839058,
      "grad_norm": 10.860318387319781,
      "learning_rate": 8.379036094136263e-08,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.40625,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.000885009765625,
      "loss": 0.3893,
      "rewards/accuracies": 0.8849999308586121,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 1.21875,
      "rewards/rejected": -2.765625,
      "step": 3860
    },
    {
      "epoch": 7.595682041216879,
      "grad_norm": 9.943400640186944,
      "learning_rate": 8.251359337329764e-08,
      "logits/chosen": -1.375,
      "logits/rejected": -1.390625,
      "logps/chosen": -508.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.00090789794921875,
      "logps/rejected": -588.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000911712646484375,
      "loss": 0.3739,
      "rewards/accuracies": 0.8400000333786011,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 1.046875,
      "rewards/rejected": -2.640625,
      "step": 3870
    },
    {
      "epoch": 7.615309126594701,
      "grad_norm": 10.803806404493619,
      "learning_rate": 8.124470273593476e-08,
      "logits/chosen": -1.4296875,
      "logits/rejected": -1.4453125,
      "logps/chosen": -572.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00081634521484375,
      "logps/rejected": -656.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000823974609375,
      "loss": 0.3599,
      "rewards/accuracies": 0.8799999356269836,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -3.078125,
      "step": 3880
    },
    {
      "epoch": 7.634936211972522,
      "grad_norm": 11.581859633592725,
      "learning_rate": 7.998374870554173e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.40625,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000926971435546875,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00093841552734375,
      "loss": 0.371,
      "rewards/accuracies": 0.8700000643730164,
      "rewards/chosen": -1.59375,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -2.796875,
      "step": 3890
    },
    {
      "epoch": 7.654563297350343,
      "grad_norm": 11.434796576304434,
      "learning_rate": 7.873079058512522e-08,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4375,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0009307861328125,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0009307861328125,
      "loss": 0.381,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": -1.6875,
      "rewards/margins": 1.078125,
      "rewards/rejected": -2.765625,
      "step": 3900
    },
    {
      "epoch": 7.654563297350343,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -580.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.0008697509765625,
      "eval_logps/rejected": -600.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.000843048095703125,
      "eval_loss": 0.5948278903961182,
      "eval_rewards/accuracies": 0.7164179086685181,
      "eval_rewards/chosen": -1.8671875,
      "eval_rewards/margins": 0.76953125,
      "eval_rewards/rejected": -2.625,
      "eval_runtime": 107.9543,
      "eval_samples_per_second": 18.526,
      "eval_steps_per_second": 0.621,
      "step": 3900
    },
    {
      "epoch": 7.674190382728165,
      "grad_norm": 10.872331131797628,
      "learning_rate": 7.74858873016424e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000766754150390625,
      "logps/rejected": -624.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000751495361328125,
      "loss": 0.3685,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.625,
      "rewards/margins": 1.328125,
      "rewards/rejected": -2.953125,
      "step": 3910
    },
    {
      "epoch": 7.693817468105986,
      "grad_norm": 9.60822682942007,
      "learning_rate": 7.624909740322905e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4453125,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000850677490234375,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000835418701171875,
      "loss": 0.3489,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.640625,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -2.984375,
      "step": 3920
    },
    {
      "epoch": 7.713444553483807,
      "grad_norm": 11.059899700049831,
      "learning_rate": 7.502047905644651e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000843048095703125,
      "loss": 0.3623,
      "rewards/accuracies": 0.8850000500679016,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 1.265625,
      "rewards/rejected": -2.78125,
      "step": 3930
    },
    {
      "epoch": 7.733071638861629,
      "grad_norm": 12.140940017863466,
      "learning_rate": 7.380009004354559e-08,
      "logits/chosen": -1.375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000881195068359375,
      "logps/rejected": -632.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00089263916015625,
      "loss": 0.3738,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 1.296875,
      "rewards/rejected": -2.953125,
      "step": 3940
    },
    {
      "epoch": 7.75269872423945,
      "grad_norm": 10.599955305219144,
      "learning_rate": 7.258798775974956e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -548.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000850677490234375,
      "logps/rejected": -636.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000865936279296875,
      "loss": 0.3768,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -1.6875,
      "rewards/margins": 1.15625,
      "rewards/rejected": -2.84375,
      "step": 3950
    },
    {
      "epoch": 7.772325809617271,
      "grad_norm": 12.259829312485664,
      "learning_rate": 7.138422921055437e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.390625,
      "logps/chosen": -488.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.000881195068359375,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000896453857421875,
      "loss": 0.3697,
      "rewards/accuracies": 0.9100000262260437,
      "rewards/chosen": -1.4375,
      "rewards/margins": 1.34375,
      "rewards/rejected": -2.78125,
      "step": 3960
    },
    {
      "epoch": 7.791952894995093,
      "grad_norm": 11.673587497917335,
      "learning_rate": 7.018887100904813e-08,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.4296875,
      "logps/chosen": -510.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000820159912109375,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00081634521484375,
      "loss": 0.3654,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -2.671875,
      "step": 3970
    },
    {
      "epoch": 7.8115799803729145,
      "grad_norm": 11.23626669760548,
      "learning_rate": 6.900196937324814e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -628.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0008392333984375,
      "loss": 0.3572,
      "rewards/accuracies": 0.8850000500679016,
      "rewards/chosen": -1.5234375,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.796875,
      "step": 3980
    },
    {
      "epoch": 7.8312070657507356,
      "grad_norm": 10.633921984388468,
      "learning_rate": 6.782358012345715e-08,
      "logits/chosen": -1.4296875,
      "logits/rejected": -1.453125,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.000865936279296875,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.625,
      "logps/rejected_top_tokens": -0.000835418701171875,
      "loss": 0.3668,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": -1.59375,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.859375,
      "step": 3990
    },
    {
      "epoch": 7.8508341511285575,
      "grad_norm": 10.65643493105821,
      "learning_rate": 6.665375867963832e-08,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.390625,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0009918212890625,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.0009613037109375,
      "loss": 0.3639,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -2.921875,
      "step": 4000
    },
    {
      "epoch": 7.8508341511285575,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -580.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.00090789794921875,
      "eval_logps/rejected": -600.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.0008697509765625,
      "eval_loss": 0.5949731469154358,
      "eval_rewards/accuracies": 0.7194030284881592,
      "eval_rewards/chosen": -1.8671875,
      "eval_rewards/margins": 0.7578125,
      "eval_rewards/rejected": -2.625,
      "eval_runtime": 108.0285,
      "eval_samples_per_second": 18.514,
      "eval_steps_per_second": 0.62,
      "step": 4000
    },
    {
      "epoch": 7.870461236506379,
      "grad_norm": 15.401979464151776,
      "learning_rate": 6.549256005880829e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4453125,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0009613037109375,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000942230224609375,
      "loss": 0.3913,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.75,
      "rewards/margins": 1.203125,
      "rewards/rejected": -2.953125,
      "step": 4010
    },
    {
      "epoch": 7.8900883218842,
      "grad_norm": 12.728403003073126,
      "learning_rate": 6.434003887245035e-08,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -576.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0009002685546875,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000885009765625,
      "loss": 0.3874,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.71875,
      "rewards/margins": 1.015625,
      "rewards/rejected": -2.734375,
      "step": 4020
    },
    {
      "epoch": 7.909715407262022,
      "grad_norm": 10.70067975910462,
      "learning_rate": 6.319624932394538e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.40625,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0008544921875,
      "logps/rejected": -584.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0008392333984375,
      "loss": 0.3847,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.625,
      "rewards/margins": 1.0546875,
      "rewards/rejected": -2.6875,
      "step": 4030
    },
    {
      "epoch": 7.929342492639843,
      "grad_norm": 13.268929321468441,
      "learning_rate": 6.206124520602327e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -564.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000881195068359375,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000835418701171875,
      "loss": 0.3719,
      "rewards/accuracies": 0.8199999928474426,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 1.1875,
      "rewards/rejected": -2.78125,
      "step": 4040
    },
    {
      "epoch": 7.948969578017664,
      "grad_norm": 16.223576138264168,
      "learning_rate": 6.093507989823252e-08,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00080108642578125,
      "logps/rejected": -628.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000804901123046875,
      "loss": 0.3576,
      "rewards/accuracies": 0.85999995470047,
      "rewards/chosen": -1.578125,
      "rewards/margins": 1.328125,
      "rewards/rejected": -2.90625,
      "step": 4050
    },
    {
      "epoch": 7.968596663395486,
      "grad_norm": 10.20306343751008,
      "learning_rate": 5.981780636443026e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000850677490234375,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000835418701171875,
      "loss": 0.3692,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.546875,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -2.796875,
      "step": 4060
    },
    {
      "epoch": 7.988223748773307,
      "grad_norm": 11.559562412684313,
      "learning_rate": 5.8709477150290786e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.00086212158203125,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.000873565673828125,
      "loss": 0.3661,
      "rewards/accuracies": 0.8449999690055847,
      "rewards/chosen": -1.640625,
      "rewards/margins": 1.0546875,
      "rewards/rejected": -2.6875,
      "step": 4070
    },
    {
      "epoch": 8.007850834151128,
      "grad_norm": 10.95225695720274,
      "learning_rate": 5.7610144380835003e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -576.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000865936279296875,
      "logps/rejected": -644.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0008544921875,
      "loss": 0.3756,
      "rewards/accuracies": 0.869999885559082,
      "rewards/chosen": -1.65625,
      "rewards/margins": 1.078125,
      "rewards/rejected": -2.734375,
      "step": 4080
    },
    {
      "epoch": 8.02747791952895,
      "grad_norm": 10.92812643521563,
      "learning_rate": 5.651985975797832e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00087738037109375,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000881195068359375,
      "loss": 0.3575,
      "rewards/accuracies": 0.8450000882148743,
      "rewards/chosen": -1.59375,
      "rewards/margins": 1.171875,
      "rewards/rejected": -2.765625,
      "step": 4090
    },
    {
      "epoch": 8.047105004906772,
      "grad_norm": 13.88801874596346,
      "learning_rate": 5.543867455809942e-08,
      "logits/chosen": -1.375,
      "logits/rejected": -1.40625,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000881195068359375,
      "logps/rejected": -632.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0008697509765625,
      "loss": 0.3563,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.546875,
      "rewards/margins": 1.4453125,
      "rewards/rejected": -2.984375,
      "step": 4100
    },
    {
      "epoch": 8.047105004906772,
      "eval_logits/chosen": -1.3984375,
      "eval_logits/rejected": -1.4296875,
      "eval_logps/chosen": -576.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.000885009765625,
      "eval_logps/rejected": -596.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.000850677490234375,
      "eval_loss": 0.5938891768455505,
      "eval_rewards/accuracies": 0.707462728023529,
      "eval_rewards/chosen": -1.828125,
      "eval_rewards/margins": 0.75390625,
      "eval_rewards/rejected": -2.578125,
      "eval_runtime": 108.0792,
      "eval_samples_per_second": 18.505,
      "eval_steps_per_second": 0.62,
      "step": 4100
    },
    {
      "epoch": 8.066732090284592,
      "grad_norm": 11.318833746867718,
      "learning_rate": 5.436663962962884e-08,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00079345703125,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00078582763671875,
      "loss": 0.3514,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.59375,
      "rewards/margins": 1.1796875,
      "rewards/rejected": -2.765625,
      "step": 4110
    },
    {
      "epoch": 8.086359175662414,
      "grad_norm": 13.189282134725309,
      "learning_rate": 5.330380539065718e-08,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.4140625,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000904083251953125,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000911712646484375,
      "loss": 0.3489,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.625,
      "rewards/margins": 1.34375,
      "rewards/rejected": -2.96875,
      "step": 4120
    },
    {
      "epoch": 8.105986261040236,
      "grad_norm": 12.834326838400363,
      "learning_rate": 5.2250221826564385e-08,
      "logits/chosen": -1.375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -556.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00092315673828125,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000888824462890625,
      "loss": 0.355,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 1.203125,
      "rewards/rejected": -2.796875,
      "step": 4130
    },
    {
      "epoch": 8.125613346418056,
      "grad_norm": 14.809258900222034,
      "learning_rate": 5.120593848766841e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.40625,
      "logps/chosen": -502.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000885009765625,
      "logps/rejected": -628.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000919342041015625,
      "loss": 0.3752,
      "rewards/accuracies": 0.8450000882148743,
      "rewards/chosen": -1.625,
      "rewards/margins": 1.2421875,
      "rewards/rejected": -2.859375,
      "step": 4140
    },
    {
      "epoch": 8.145240431795878,
      "grad_norm": 10.507531827005831,
      "learning_rate": 5.017100448689538e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000904083251953125,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0009307861328125,
      "loss": 0.344,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.59375,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.859375,
      "step": 4150
    },
    {
      "epoch": 8.1648675171737,
      "grad_norm": 12.259586907707716,
      "learning_rate": 4.914546849746923e-08,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.4453125,
      "logps/chosen": -548.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000812530517578125,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000823974609375,
      "loss": 0.3529,
      "rewards/accuracies": 0.8450000882148743,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 1.25,
      "rewards/rejected": -2.84375,
      "step": 4160
    },
    {
      "epoch": 8.18449460255152,
      "grad_norm": 12.329389242214374,
      "learning_rate": 4.812937875062328e-08,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000904083251953125,
      "loss": 0.3521,
      "rewards/accuracies": 0.8700000643730164,
      "rewards/chosen": -1.59375,
      "rewards/margins": 1.21875,
      "rewards/rejected": -2.8125,
      "step": 4170
    },
    {
      "epoch": 8.204121687929343,
      "grad_norm": 11.291299154109046,
      "learning_rate": 4.71227830333312e-08,
      "logits/chosen": -1.4296875,
      "logits/rejected": -1.453125,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0008087158203125,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.00080108642578125,
      "loss": 0.3418,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -2.984375,
      "step": 4180
    },
    {
      "epoch": 8.223748773307165,
      "grad_norm": 10.114702294210609,
      "learning_rate": 4.612572868605999e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4375,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000812530517578125,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000797271728515625,
      "loss": 0.3527,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.75,
      "rewards/margins": 1.171875,
      "rewards/rejected": -2.921875,
      "step": 4190
    },
    {
      "epoch": 8.243375858684985,
      "grad_norm": 11.685557169535775,
      "learning_rate": 4.513826260054357e-08,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000926971435546875,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0009307861328125,
      "loss": 0.3484,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 1.34375,
      "rewards/rejected": -2.96875,
      "step": 4200
    },
    {
      "epoch": 8.243375858684985,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -580.0,
      "eval_logps/chosen_bottom_tokens": -14.0,
      "eval_logps/chosen_top_tokens": -0.000888824462890625,
      "eval_logps/rejected": -600.0,
      "eval_logps/rejected_bottom_tokens": -13.875,
      "eval_logps/rejected_top_tokens": -0.00084686279296875,
      "eval_loss": 0.5968652367591858,
      "eval_rewards/accuracies": 0.7044776082038879,
      "eval_rewards/chosen": -1.875,
      "eval_rewards/margins": 0.765625,
      "eval_rewards/rejected": -2.640625,
      "eval_runtime": 107.9539,
      "eval_samples_per_second": 18.526,
      "eval_steps_per_second": 0.621,
      "step": 4200
    },
    {
      "epoch": 8.263002944062807,
      "grad_norm": 11.223061664314738,
      "learning_rate": 4.416043121757715e-08,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.4375,
      "logps/chosen": -580.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00083160400390625,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.000835418701171875,
      "loss": 0.3615,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.734375,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -2.921875,
      "step": 4210
    },
    {
      "epoch": 8.282630029440629,
      "grad_norm": 13.979364868218873,
      "learning_rate": 4.319228052483348e-08,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.4375,
      "logps/chosen": -510.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000812530517578125,
      "logps/rejected": -580.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00079345703125,
      "loss": 0.3312,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.421875,
      "rewards/margins": 1.296875,
      "rewards/rejected": -2.71875,
      "step": 4220
    },
    {
      "epoch": 8.302257114818449,
      "grad_norm": 13.716554098357632,
      "learning_rate": 4.223385605469962e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.3984375,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000885009765625,
      "logps/rejected": -636.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.0008544921875,
      "loss": 0.3492,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": -1.5625,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -2.890625,
      "step": 4230
    },
    {
      "epoch": 8.321884200196271,
      "grad_norm": 12.209448291726472,
      "learning_rate": 4.128520288213608e-08,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.453125,
      "logps/chosen": -548.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000934600830078125,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00093841552734375,
      "loss": 0.3511,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -2.984375,
      "step": 4240
    },
    {
      "epoch": 8.341511285574093,
      "grad_norm": 15.422956351720499,
      "learning_rate": 4.0346365622556276e-08,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00083160400390625,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00081634521484375,
      "loss": 0.3502,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.5625,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -2.828125,
      "step": 4250
    },
    {
      "epoch": 8.361138370951913,
      "grad_norm": 10.987525709594365,
      "learning_rate": 3.9417388429728884e-08,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000926971435546875,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.00092315673828125,
      "loss": 0.3449,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 1.375,
      "rewards/rejected": -2.984375,
      "step": 4260
    },
    {
      "epoch": 8.380765456329735,
      "grad_norm": 13.277350901991092,
      "learning_rate": 3.8498314993700764e-08,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.390625,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.000957489013671875,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.000946044921875,
      "loss": 0.3563,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.640625,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -2.9375,
      "step": 4270
    },
    {
      "epoch": 8.400392541707557,
      "grad_norm": 11.773506804599066,
      "learning_rate": 3.7589188538742654e-08,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -572.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00079345703125,
      "logps/rejected": -628.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000804901123046875,
      "loss": 0.3633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.625,
      "rewards/margins": 1.34375,
      "rewards/rejected": -2.96875,
      "step": 4280
    },
    {
      "epoch": 8.420019627085377,
      "grad_norm": 12.35899932980167,
      "learning_rate": 3.66900518213159e-08,
      "logits/chosen": -1.4296875,
      "logits/rejected": -1.453125,
      "logps/chosen": -576.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000843048095703125,
      "logps/rejected": -648.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0009765625,
      "loss": 0.3589,
      "rewards/accuracies": 0.9050000309944153,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -2.890625,
      "step": 4290
    },
    {
      "epoch": 8.4396467124632,
      "grad_norm": 10.566216182379264,
      "learning_rate": 3.580094712806195e-08,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.00090789794921875,
      "logps/rejected": -636.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000919342041015625,
      "loss": 0.3359,
      "rewards/accuracies": 0.8749998807907104,
      "rewards/chosen": -1.609375,
      "rewards/margins": 1.3828125,
      "rewards/rejected": -2.984375,
      "step": 4300
    },
    {
      "epoch": 8.4396467124632,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -580.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.00087738037109375,
      "eval_logps/rejected": -604.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.00086212158203125,
      "eval_loss": 0.5965551733970642,
      "eval_rewards/accuracies": 0.7044776082038879,
      "eval_rewards/chosen": -1.8828125,
      "eval_rewards/margins": 0.7734375,
      "eval_rewards/rejected": -2.65625,
      "eval_runtime": 108.2337,
      "eval_samples_per_second": 18.479,
      "eval_steps_per_second": 0.619,
      "step": 4300
    },
    {
      "epoch": 8.459273797841021,
      "grad_norm": 12.376710994660526,
      "learning_rate": 3.49219162738133e-08,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.4140625,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00095367431640625,
      "logps/rejected": -576.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000926971435546875,
      "loss": 0.371,
      "rewards/accuracies": 0.8200000524520874,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 1.046875,
      "rewards/rejected": -2.671875,
      "step": 4310
    },
    {
      "epoch": 8.478900883218841,
      "grad_norm": 10.141612269870464,
      "learning_rate": 3.405300059962729e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -576.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00087738037109375,
      "logps/rejected": -660.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000850677490234375,
      "loss": 0.3598,
      "rewards/accuracies": 0.8149999380111694,
      "rewards/chosen": -1.859375,
      "rewards/margins": 1.125,
      "rewards/rejected": -2.984375,
      "step": 4320
    },
    {
      "epoch": 8.498527968596663,
      "grad_norm": 13.581768077877944,
      "learning_rate": 3.319424097084153e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.390625,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0009002685546875,
      "logps/rejected": -676.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000942230224609375,
      "loss": 0.3476,
      "rewards/accuracies": 0.8850000500679016,
      "rewards/chosen": -1.65625,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -2.9375,
      "step": 4330
    },
    {
      "epoch": 8.518155053974485,
      "grad_norm": 10.680900604075344,
      "learning_rate": 3.2345677775152014e-08,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00089263916015625,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0008697509765625,
      "loss": 0.3527,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.671875,
      "rewards/margins": 1.296875,
      "rewards/rejected": -2.96875,
      "step": 4340
    },
    {
      "epoch": 8.537782139352306,
      "grad_norm": 11.197449979240112,
      "learning_rate": 3.150735092071391e-08,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0008544921875,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000850677490234375,
      "loss": 0.3679,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -2.796875,
      "step": 4350
    },
    {
      "epoch": 8.557409224730128,
      "grad_norm": 11.967712785654316,
      "learning_rate": 3.067929983426434e-08,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.421875,
      "logps/chosen": -580.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000873565673828125,
      "logps/rejected": -636.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000843048095703125,
      "loss": 0.3536,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7578125,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -3.015625,
      "step": 4360
    },
    {
      "epoch": 8.57703631010795,
      "grad_norm": 11.095146468888284,
      "learning_rate": 2.9861563459268433e-08,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.40625,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0008697509765625,
      "logps/rejected": -624.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.000873565673828125,
      "loss": 0.355,
      "rewards/accuracies": 0.8650000691413879,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -2.984375,
      "step": 4370
    },
    {
      "epoch": 8.59666339548577,
      "grad_norm": 13.206766651368518,
      "learning_rate": 2.9054180254087512e-08,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -580.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.0008087158203125,
      "logps/rejected": -656.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.000820159912109375,
      "loss": 0.365,
      "rewards/accuracies": 0.8650000691413879,
      "rewards/chosen": -1.6875,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -2.875,
      "step": 4380
    },
    {
      "epoch": 8.616290480863592,
      "grad_norm": 9.29695813066069,
      "learning_rate": 2.8257188190170785e-08,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -608.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000789642333984375,
      "logps/rejected": -688.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000827789306640625,
      "loss": 0.3489,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.671875,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -2.90625,
      "step": 4390
    },
    {
      "epoch": 8.635917566241414,
      "grad_norm": 13.112622948489534,
      "learning_rate": 2.7470624750269013e-08,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.421875,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000827789306640625,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.000823974609375,
      "loss": 0.3639,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": -1.578125,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -2.875,
      "step": 4400
    },
    {
      "epoch": 8.635917566241414,
      "eval_logits/chosen": -1.3984375,
      "eval_logits/rejected": -1.4296875,
      "eval_logps/chosen": -580.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.0009002685546875,
      "eval_logps/rejected": -596.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.0008697509765625,
      "eval_loss": 0.5978894233703613,
      "eval_rewards/accuracies": 0.707462728023529,
      "eval_rewards/chosen": -1.8515625,
      "eval_rewards/margins": 0.74609375,
      "eval_rewards/rejected": -2.59375,
      "eval_runtime": 108.1117,
      "eval_samples_per_second": 18.499,
      "eval_steps_per_second": 0.62,
      "step": 4400
    },
    {
      "epoch": 8.655544651619234,
      "grad_norm": 14.045254103637847,
      "learning_rate": 2.669452692667218e-08,
      "logits/chosen": -1.453125,
      "logits/rejected": -1.46875,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0008087158203125,
      "logps/rejected": -628.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00081634521484375,
      "loss": 0.3671,
      "rewards/accuracies": 0.8400000333786011,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 1.09375,
      "rewards/rejected": -2.734375,
      "step": 4410
    },
    {
      "epoch": 8.675171736997056,
      "grad_norm": 13.183615803977235,
      "learning_rate": 2.5928931219469348e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000835418701171875,
      "loss": 0.3569,
      "rewards/accuracies": 0.8700000643730164,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -2.84375,
      "step": 4420
    },
    {
      "epoch": 8.694798822374878,
      "grad_norm": 11.9847649039614,
      "learning_rate": 2.517387363483242e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -556.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00083160400390625,
      "logps/rejected": -668.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00110626220703125,
      "loss": 0.3644,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5078125,
      "rewards/margins": 1.4375,
      "rewards/rejected": -2.9375,
      "step": 4430
    },
    {
      "epoch": 8.714425907752698,
      "grad_norm": 11.758750789803823,
      "learning_rate": 2.4429389683322394e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -506.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.00090789794921875,
      "logps/rejected": -628.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000942230224609375,
      "loss": 0.347,
      "rewards/accuracies": 0.9050000309944153,
      "rewards/chosen": -1.53125,
      "rewards/margins": 1.453125,
      "rewards/rejected": -2.984375,
      "step": 4440
    },
    {
      "epoch": 8.73405299313052,
      "grad_norm": 12.063528763672434,
      "learning_rate": 2.3695514378219593e-08,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.40625,
      "logps/chosen": -584.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00092315673828125,
      "logps/rejected": -628.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000888824462890625,
      "loss": 0.3368,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 1.328125,
      "rewards/rejected": -2.890625,
      "step": 4450
    },
    {
      "epoch": 8.753680078508342,
      "grad_norm": 12.302966216794538,
      "learning_rate": 2.297228223387673e-08,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.00089263916015625,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00091552734375,
      "loss": 0.3624,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.59375,
      "rewards/margins": 1.328125,
      "rewards/rejected": -2.921875,
      "step": 4460
    },
    {
      "epoch": 8.773307163886162,
      "grad_norm": 12.416744994343894,
      "learning_rate": 2.225972726409586e-08,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.375,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.001007080078125,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.001007080078125,
      "loss": 0.3627,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": -1.640625,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -2.84375,
      "step": 4470
    },
    {
      "epoch": 8.792934249263984,
      "grad_norm": 10.542790739232196,
      "learning_rate": 2.155788298052874e-08,
      "logits/chosen": -1.375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -588.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00086212158203125,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0008087158203125,
      "loss": 0.3515,
      "rewards/accuracies": 0.85999995470047,
      "rewards/chosen": -1.734375,
      "rewards/margins": 1.1171875,
      "rewards/rejected": -2.84375,
      "step": 4480
    },
    {
      "epoch": 8.812561334641806,
      "grad_norm": 10.095206706315418,
      "learning_rate": 2.0866782391100484e-08,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.390625,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000827789306640625,
      "logps/rejected": -648.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.0008392333984375,
      "loss": 0.3599,
      "rewards/accuracies": 0.8799999356269836,
      "rewards/chosen": -1.671875,
      "rewards/margins": 1.296875,
      "rewards/rejected": -2.96875,
      "step": 4490
    },
    {
      "epoch": 8.832188420019627,
      "grad_norm": 10.809244304846091,
      "learning_rate": 2.0186457998457613e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0009002685546875,
      "logps/rejected": -636.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000881195068359375,
      "loss": 0.3563,
      "rewards/accuracies": 0.9199999570846558,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 1.40625,
      "rewards/rejected": -2.953125,
      "step": 4500
    },
    {
      "epoch": 8.832188420019627,
      "eval_logits/chosen": -1.3984375,
      "eval_logits/rejected": -1.4296875,
      "eval_logps/chosen": -580.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.000904083251953125,
      "eval_logps/rejected": -600.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.000873565673828125,
      "eval_loss": 0.5979223847389221,
      "eval_rewards/accuracies": 0.707462728023529,
      "eval_rewards/chosen": -1.859375,
      "eval_rewards/margins": 0.76171875,
      "eval_rewards/rejected": -2.625,
      "eval_runtime": 108.3131,
      "eval_samples_per_second": 18.465,
      "eval_steps_per_second": 0.619,
      "step": 4500
    },
    {
      "epoch": 8.851815505397449,
      "grad_norm": 10.472003628252363,
      "learning_rate": 1.9516941798438935e-08,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.421875,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00084686279296875,
      "loss": 0.3525,
      "rewards/accuracies": 0.8550001382827759,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 1.21875,
      "rewards/rejected": -2.90625,
      "step": 4510
    },
    {
      "epoch": 8.87144259077527,
      "grad_norm": 14.197024509603557,
      "learning_rate": 1.8858265278571346e-08,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3828125,
      "logps/chosen": -512.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0009918212890625,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00104522705078125,
      "loss": 0.35,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.703125,
      "rewards/margins": 1.1875,
      "rewards/rejected": -2.890625,
      "step": 4520
    },
    {
      "epoch": 8.89106967615309,
      "grad_norm": 11.730282187199597,
      "learning_rate": 1.821045941658847e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.40625,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00080108642578125,
      "logps/rejected": -624.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000782012939453125,
      "loss": 0.3531,
      "rewards/accuracies": 0.8550000190734863,
      "rewards/chosen": -1.6875,
      "rewards/margins": 1.171875,
      "rewards/rejected": -2.859375,
      "step": 4530
    },
    {
      "epoch": 8.910696761530913,
      "grad_norm": 12.183920255550538,
      "learning_rate": 1.757355467897409e-08,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0008697509765625,
      "logps/rejected": -624.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0008544921875,
      "loss": 0.3586,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.859375,
      "step": 4540
    },
    {
      "epoch": 8.930323846908735,
      "grad_norm": 16.902678047586438,
      "learning_rate": 1.694758101952909e-08,
      "logits/chosen": -1.375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -500.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00087738037109375,
      "logps/rejected": -572.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.00083160400390625,
      "loss": 0.3646,
      "rewards/accuracies": 0.8450000882148743,
      "rewards/chosen": -1.5625,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.84375,
      "step": 4550
    },
    {
      "epoch": 8.949950932286555,
      "grad_norm": 11.768620827284108,
      "learning_rate": 1.6332567877962887e-08,
      "logits/chosen": -1.375,
      "logits/rejected": -1.40625,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -14.0625,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0008544921875,
      "loss": 0.351,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.625,
      "rewards/margins": 1.3359375,
      "rewards/rejected": -2.953125,
      "step": 4560
    },
    {
      "epoch": 8.969578017664377,
      "grad_norm": 13.962472982347483,
      "learning_rate": 1.5728544178508744e-08,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.375,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000957489013671875,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0009613037109375,
      "loss": 0.3648,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5546875,
      "rewards/margins": 1.265625,
      "rewards/rejected": -2.8125,
      "step": 4570
    },
    {
      "epoch": 8.989205103042199,
      "grad_norm": 11.885259392354866,
      "learning_rate": 1.513553832856357e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -564.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0008697509765625,
      "logps/rejected": -652.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00086212158203125,
      "loss": 0.3796,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -2.984375,
      "step": 4580
    },
    {
      "epoch": 9.008832188420019,
      "grad_norm": 13.913153579158871,
      "learning_rate": 1.455357821735173e-08,
      "logits/chosen": -1.375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00093841552734375,
      "logps/rejected": -628.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0009613037109375,
      "loss": 0.35,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 1.40625,
      "rewards/rejected": -2.953125,
      "step": 4590
    },
    {
      "epoch": 9.028459273797841,
      "grad_norm": 15.123594232623052,
      "learning_rate": 1.3982691214613678e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.0009307861328125,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00092315673828125,
      "loss": 0.353,
      "rewards/accuracies": 0.8349999189376831,
      "rewards/chosen": -1.65625,
      "rewards/margins": 1.171875,
      "rewards/rejected": -2.828125,
      "step": 4600
    },
    {
      "epoch": 9.028459273797841,
      "eval_logits/chosen": -1.3984375,
      "eval_logits/rejected": -1.4296875,
      "eval_logps/chosen": -580.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.00087738037109375,
      "eval_logps/rejected": -600.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.000843048095703125,
      "eval_loss": 0.5981213450431824,
      "eval_rewards/accuracies": 0.6985074877738953,
      "eval_rewards/chosen": -1.8671875,
      "eval_rewards/margins": 0.76171875,
      "eval_rewards/rejected": -2.625,
      "eval_runtime": 108.308,
      "eval_samples_per_second": 18.466,
      "eval_steps_per_second": 0.619,
      "step": 4600
    },
    {
      "epoch": 9.048086359175663,
      "grad_norm": 11.670476508295476,
      "learning_rate": 1.3422904169318544e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000797271728515625,
      "logps/rejected": -624.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000797271728515625,
      "loss": 0.3413,
      "rewards/accuracies": 0.89000004529953,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 1.359375,
      "rewards/rejected": -2.96875,
      "step": 4610
    },
    {
      "epoch": 9.067713444553483,
      "grad_norm": 12.360606000046072,
      "learning_rate": 1.2874243408401376e-08,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.4140625,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.000873565673828125,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.5625,
      "logps/rejected_top_tokens": -0.000850677490234375,
      "loss": 0.3356,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.5,
      "rewards/margins": 1.28125,
      "rewards/rejected": -2.78125,
      "step": 4620
    },
    {
      "epoch": 9.087340529931305,
      "grad_norm": 12.10726086361965,
      "learning_rate": 1.2336734735525268e-08,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.453125,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00083160400390625,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000820159912109375,
      "loss": 0.3409,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.8125,
      "step": 4630
    },
    {
      "epoch": 9.106967615309127,
      "grad_norm": 11.696000638859102,
      "learning_rate": 1.1810403429867445e-08,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00092315673828125,
      "logps/rejected": -640.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000911712646484375,
      "loss": 0.3481,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.75,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -2.921875,
      "step": 4640
    },
    {
      "epoch": 9.126594700686947,
      "grad_norm": 11.124831553826729,
      "learning_rate": 1.129527424493068e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.3984375,
      "logps/chosen": -548.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -632.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.3542,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 1.265625,
      "rewards/rejected": -2.9375,
      "step": 4650
    },
    {
      "epoch": 9.14622178606477,
      "grad_norm": 13.099416868432309,
      "learning_rate": 1.0791371407378902e-08,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.453125,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -596.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.000850677490234375,
      "loss": 0.3591,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.921875,
      "step": 4660
    },
    {
      "epoch": 9.165848871442591,
      "grad_norm": 15.476496404936588,
      "learning_rate": 1.0298718615897983e-08,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.40625,
      "logps/chosen": -516.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0009002685546875,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000858306884765625,
      "loss": 0.3592,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.546875,
      "rewards/margins": 1.328125,
      "rewards/rejected": -2.875,
      "step": 4670
    },
    {
      "epoch": 9.185475956820412,
      "grad_norm": 11.22918157402849,
      "learning_rate": 9.817339040081002e-09,
      "logits/chosen": -1.375,
      "logits/rejected": -1.40625,
      "logps/chosen": -512.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -576.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000888824462890625,
      "loss": 0.3376,
      "rewards/accuracies": 0.880000114440918,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 1.203125,
      "rewards/rejected": -2.8125,
      "step": 4680
    },
    {
      "epoch": 9.205103042198234,
      "grad_norm": 11.853639781559089,
      "learning_rate": 9.347255319338804e-09,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000888824462890625,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0008697509765625,
      "loss": 0.3483,
      "rewards/accuracies": 0.9199999570846558,
      "rewards/chosen": -1.5625,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -2.96875,
      "step": 4690
    },
    {
      "epoch": 9.224730127576056,
      "grad_norm": 10.9646558915127,
      "learning_rate": 8.888489561835022e-09,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0008697509765625,
      "logps/rejected": -636.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.000858306884765625,
      "loss": 0.3514,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -2.984375,
      "step": 4700
    },
    {
      "epoch": 9.224730127576056,
      "eval_logits/chosen": -1.3984375,
      "eval_logits/rejected": -1.4296875,
      "eval_logps/chosen": -580.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.00086212158203125,
      "eval_logps/rejected": -600.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.000827789306640625,
      "eval_loss": 0.5979077219963074,
      "eval_rewards/accuracies": 0.6985074877738953,
      "eval_rewards/chosen": -1.859375,
      "eval_rewards/margins": 0.765625,
      "eval_rewards/rejected": -2.625,
      "eval_runtime": 108.2657,
      "eval_samples_per_second": 18.473,
      "eval_steps_per_second": 0.619,
      "step": 4700
    },
    {
      "epoch": 9.244357212953876,
      "grad_norm": 10.122903531296545,
      "learning_rate": 8.441063343446548e-09,
      "logits/chosen": -1.4375,
      "logits/rejected": -1.4453125,
      "logps/chosen": -482.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -588.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00086212158203125,
      "loss": 0.3486,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.890625,
      "step": 4710
    },
    {
      "epoch": 9.263984298331698,
      "grad_norm": 11.909843897950127,
      "learning_rate": 8.00499770674859e-09,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -564.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00091552734375,
      "logps/rejected": -644.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000904083251953125,
      "loss": 0.3487,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.9375,
      "step": 4720
    },
    {
      "epoch": 9.28361138370952,
      "grad_norm": 11.711069712684242,
      "learning_rate": 7.58031316002522e-09,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.4140625,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.000804901123046875,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0007781982421875,
      "loss": 0.3634,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.609375,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.875,
      "step": 4730
    },
    {
      "epoch": 9.30323846908734,
      "grad_norm": 10.862219564432905,
      "learning_rate": 7.167029676304781e-09,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -548.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.000873565673828125,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.0008697509765625,
      "loss": 0.3465,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -2.875,
      "step": 4740
    },
    {
      "epoch": 9.322865554465162,
      "grad_norm": 13.451953701239999,
      "learning_rate": 6.7651666924204984e-09,
      "logits/chosen": -1.375,
      "logits/rejected": -1.390625,
      "logps/chosen": -496.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000926971435546875,
      "logps/rejected": -564.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00090789794921875,
      "loss": 0.3474,
      "rewards/accuracies": 0.8799999356269836,
      "rewards/chosen": -1.609375,
      "rewards/margins": 1.28125,
      "rewards/rejected": -2.890625,
      "step": 4750
    },
    {
      "epoch": 9.342492639842984,
      "grad_norm": 15.228199482134903,
      "learning_rate": 6.374743108096547e-09,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.3984375,
      "logps/chosen": -508.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000881195068359375,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00087738037109375,
      "loss": 0.359,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 1.28125,
      "rewards/rejected": -2.875,
      "step": 4760
    },
    {
      "epoch": 9.362119725220804,
      "grad_norm": 11.009933240515391,
      "learning_rate": 5.9957772850589564e-09,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.3984375,
      "logps/chosen": -556.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00090789794921875,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00087738037109375,
      "loss": 0.356,
      "rewards/accuracies": 0.8650000691413879,
      "rewards/chosen": -1.59375,
      "rewards/margins": 1.328125,
      "rewards/rejected": -2.90625,
      "step": 4770
    },
    {
      "epoch": 9.381746810598626,
      "grad_norm": 10.277894448124382,
      "learning_rate": 5.628287046172187e-09,
      "logits/chosen": -1.3515625,
      "logits/rejected": -1.375,
      "logps/chosen": -528.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000835418701171875,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000835418701171875,
      "loss": 0.3541,
      "rewards/accuracies": 0.8850000500679016,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 1.296875,
      "rewards/rejected": -2.890625,
      "step": 4780
    },
    {
      "epoch": 9.401373895976448,
      "grad_norm": 13.874643137091146,
      "learning_rate": 5.272289674600916e-09,
      "logits/chosen": -1.359375,
      "logits/rejected": -1.390625,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0008087158203125,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00080108642578125,
      "loss": 0.3728,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 1.1953125,
      "rewards/rejected": -2.828125,
      "step": 4790
    },
    {
      "epoch": 9.421000981354268,
      "grad_norm": 14.853698997198226,
      "learning_rate": 4.927801912997214e-09,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -486.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.000850677490234375,
      "logps/rejected": -560.0,
      "logps/rejected_bottom_tokens": -13.625,
      "logps/rejected_top_tokens": -0.000835418701171875,
      "loss": 0.3434,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": -1.515625,
      "rewards/margins": 1.2578125,
      "rewards/rejected": -2.78125,
      "step": 4800
    },
    {
      "epoch": 9.421000981354268,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4296875,
      "eval_logps/chosen": -580.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.00087738037109375,
      "eval_logps/rejected": -600.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.00084686279296875,
      "eval_loss": 0.5972570776939392,
      "eval_rewards/accuracies": 0.7014925479888916,
      "eval_rewards/chosen": -1.8671875,
      "eval_rewards/margins": 0.765625,
      "eval_rewards/rejected": -2.640625,
      "eval_runtime": 108.376,
      "eval_samples_per_second": 18.454,
      "eval_steps_per_second": 0.618,
      "step": 4800
    },
    {
      "epoch": 9.44062806673209,
      "grad_norm": 11.155786101466646,
      "learning_rate": 4.594839962713009e-09,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.421875,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.00090789794921875,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000904083251953125,
      "loss": 0.3635,
      "rewards/accuracies": 0.8449999690055847,
      "rewards/chosen": -1.65625,
      "rewards/margins": 1.3125,
      "rewards/rejected": -2.96875,
      "step": 4810
    },
    {
      "epoch": 9.460255152109912,
      "grad_norm": 11.415347601053586,
      "learning_rate": 4.273419483038304e-09,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.4453125,
      "logps/chosen": -560.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.00080108642578125,
      "logps/rejected": -640.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00079345703125,
      "loss": 0.345,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.578125,
      "rewards/margins": 1.2109375,
      "rewards/rejected": -2.796875,
      "step": 4820
    },
    {
      "epoch": 9.479882237487733,
      "grad_norm": 10.958041240615906,
      "learning_rate": 3.963555590464601e-09,
      "logits/chosen": -1.3359375,
      "logits/rejected": -1.3828125,
      "logps/chosen": -512.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.00103759765625,
      "logps/rejected": -540.0,
      "logps/rejected_bottom_tokens": -13.625,
      "logps/rejected_top_tokens": -0.001007080078125,
      "loss": 0.3657,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -2.625,
      "step": 4830
    },
    {
      "epoch": 9.499509322865554,
      "grad_norm": 11.57554214332043,
      "learning_rate": 3.6652628579740276e-09,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.40625,
      "logps/chosen": -528.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000926971435546875,
      "logps/rejected": -604.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000919342041015625,
      "loss": 0.3403,
      "rewards/accuracies": 0.8649999499320984,
      "rewards/chosen": -1.578125,
      "rewards/margins": 1.28125,
      "rewards/rejected": -2.859375,
      "step": 4840
    },
    {
      "epoch": 9.519136408243376,
      "grad_norm": 9.297108054051924,
      "learning_rate": 3.378555314353937e-09,
      "logits/chosen": -1.375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -510.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -580.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000865936279296875,
      "loss": 0.3418,
      "rewards/accuracies": 0.8149999380111694,
      "rewards/chosen": -1.71875,
      "rewards/margins": 1.1015625,
      "rewards/rejected": -2.828125,
      "step": 4850
    },
    {
      "epoch": 9.538763493621197,
      "grad_norm": 9.130763500239256,
      "learning_rate": 3.1034464435371053e-09,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000911712646484375,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.3505,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.6875,
      "rewards/margins": 1.296875,
      "rewards/rejected": -2.984375,
      "step": 4860
    },
    {
      "epoch": 9.558390578999019,
      "grad_norm": 14.1811429926495,
      "learning_rate": 2.8399491839677103e-09,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.40625,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000911712646484375,
      "logps/rejected": -616.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.00089263916015625,
      "loss": 0.3473,
      "rewards/accuracies": 0.89000004529953,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 1.359375,
      "rewards/rejected": -2.953125,
      "step": 4870
    },
    {
      "epoch": 9.57801766437684,
      "grad_norm": 12.869889556323235,
      "learning_rate": 2.5880759279925947e-09,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.421875,
      "logps/chosen": -580.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.0008392333984375,
      "logps/rejected": -660.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000850677490234375,
      "loss": 0.3415,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.6875,
      "rewards/margins": 1.3203125,
      "rewards/rejected": -3.015625,
      "step": 4880
    },
    {
      "epoch": 9.59764474975466,
      "grad_norm": 12.548800832744114,
      "learning_rate": 2.3478385212787055e-09,
      "logits/chosen": -1.375,
      "logits/rejected": -1.390625,
      "logps/chosen": -572.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000888824462890625,
      "logps/rejected": -652.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.00086212158203125,
      "loss": 0.3548,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.671875,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -2.96875,
      "step": 4890
    },
    {
      "epoch": 9.617271835132483,
      "grad_norm": 13.023279708907932,
      "learning_rate": 2.1192482622557904e-09,
      "logits/chosen": -1.421875,
      "logits/rejected": -1.453125,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000850677490234375,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000835418701171875,
      "loss": 0.3492,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 1.2734375,
      "rewards/rejected": -2.921875,
      "step": 4900
    },
    {
      "epoch": 9.617271835132483,
      "eval_logits/chosen": -1.3984375,
      "eval_logits/rejected": -1.4296875,
      "eval_logps/chosen": -580.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.00087738037109375,
      "eval_logps/rejected": -600.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.0008392333984375,
      "eval_loss": 0.5981469750404358,
      "eval_rewards/accuracies": 0.7044776082038879,
      "eval_rewards/chosen": -1.875,
      "eval_rewards/margins": 0.7578125,
      "eval_rewards/rejected": -2.640625,
      "eval_runtime": 108.362,
      "eval_samples_per_second": 18.457,
      "eval_steps_per_second": 0.618,
      "step": 4900
    },
    {
      "epoch": 9.636898920510305,
      "grad_norm": 13.199505551719971,
      "learning_rate": 1.902315901585155e-09,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.421875,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000881195068359375,
      "logps/rejected": -652.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.3268,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.6875,
      "rewards/margins": 1.5703125,
      "rewards/rejected": -3.265625,
      "step": 4910
    },
    {
      "epoch": 9.656526005888125,
      "grad_norm": 12.666272933800087,
      "learning_rate": 1.6970516416540127e-09,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.390625,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00092315673828125,
      "logps/rejected": -612.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000934600830078125,
      "loss": 0.3479,
      "rewards/accuracies": 0.89000004529953,
      "rewards/chosen": -1.671875,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -2.96875,
      "step": 4920
    },
    {
      "epoch": 9.676153091265947,
      "grad_norm": 15.121041451718062,
      "learning_rate": 1.5034651360956718e-09,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -584.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.000926971435546875,
      "logps/rejected": -640.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.00090789794921875,
      "loss": 0.3591,
      "rewards/accuracies": 0.8399999737739563,
      "rewards/chosen": -1.8125,
      "rewards/margins": 1.1640625,
      "rewards/rejected": -2.96875,
      "step": 4930
    },
    {
      "epoch": 9.695780176643769,
      "grad_norm": 14.043067920247658,
      "learning_rate": 1.3215654893354578e-09,
      "logits/chosen": -1.375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -14.0,
      "logps/chosen_top_tokens": -0.0008544921875,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.000873565673828125,
      "loss": 0.3503,
      "rewards/accuracies": 0.8549998998641968,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 1.3125,
      "rewards/rejected": -2.90625,
      "step": 4940
    },
    {
      "epoch": 9.71540726202159,
      "grad_norm": 16.327308164510683,
      "learning_rate": 1.1513612561626362e-09,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0009002685546875,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0008697509765625,
      "loss": 0.3552,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": -1.5859375,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -2.875,
      "step": 4950
    },
    {
      "epoch": 9.735034347399411,
      "grad_norm": 11.799295708369108,
      "learning_rate": 9.928604413280694e-10,
      "logits/chosen": -1.4140625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.00083160400390625,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.000827789306640625,
      "loss": 0.3506,
      "rewards/accuracies": 0.8850000500679016,
      "rewards/chosen": -1.59375,
      "rewards/margins": 1.2265625,
      "rewards/rejected": -2.8125,
      "step": 4960
    },
    {
      "epoch": 9.754661432777233,
      "grad_norm": 12.311557663406573,
      "learning_rate": 8.460704991676004e-10,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4140625,
      "logps/chosen": -524.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.001007080078125,
      "logps/rejected": -600.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.0009918212890625,
      "loss": 0.3486,
      "rewards/accuracies": 0.8650000691413879,
      "rewards/chosen": -1.609375,
      "rewards/margins": 1.2890625,
      "rewards/rejected": -2.890625,
      "step": 4970
    },
    {
      "epoch": 9.774288518155053,
      "grad_norm": 11.77978565668884,
      "learning_rate": 7.109983332516945e-10,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.3984375,
      "logps/chosen": -516.0,
      "logps/chosen_bottom_tokens": -13.75,
      "logps/chosen_top_tokens": -0.0008697509765625,
      "logps/rejected": -608.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.0008697509765625,
      "loss": 0.3436,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 1.21875,
      "rewards/rejected": -2.875,
      "step": 4980
    },
    {
      "epoch": 9.793915603532875,
      "grad_norm": 11.06530017719301,
      "learning_rate": 5.876502960605878e-10,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.4296875,
      "logps/chosen": -552.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.00086212158203125,
      "logps/rejected": -648.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000911712646484375,
      "loss": 0.3478,
      "rewards/accuracies": 0.8700000047683716,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 1.3125,
      "rewards/rejected": -2.984375,
      "step": 4990
    },
    {
      "epoch": 9.813542688910697,
      "grad_norm": 14.260832030615752,
      "learning_rate": 4.7603218868561e-10,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.40625,
      "logps/chosen": -528.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0009307861328125,
      "logps/rejected": -636.0,
      "logps/rejected_bottom_tokens": -13.9375,
      "logps/rejected_top_tokens": -0.00092315673828125,
      "loss": 0.3487,
      "rewards/accuracies": 0.8949999809265137,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 1.21875,
      "rewards/rejected": -2.8125,
      "step": 5000
    },
    {
      "epoch": 9.813542688910697,
      "eval_logits/chosen": -1.40625,
      "eval_logits/rejected": -1.4375,
      "eval_logps/chosen": -580.0,
      "eval_logps/chosen_bottom_tokens": -13.9375,
      "eval_logps/chosen_top_tokens": -0.00086212158203125,
      "eval_logps/rejected": -600.0,
      "eval_logps/rejected_bottom_tokens": -13.8125,
      "eval_logps/rejected_top_tokens": -0.0008392333984375,
      "eval_loss": 0.5967333912849426,
      "eval_rewards/accuracies": 0.7134328484535217,
      "eval_rewards/chosen": -1.8671875,
      "eval_rewards/margins": 0.7734375,
      "eval_rewards/rejected": -2.640625,
      "eval_runtime": 108.4167,
      "eval_samples_per_second": 18.447,
      "eval_steps_per_second": 0.618,
      "step": 5000
    },
    {
      "epoch": 9.833169774288518,
      "grad_norm": 12.34808824315509,
      "learning_rate": 3.7614926055634653e-10,
      "logits/chosen": -1.3984375,
      "logits/rejected": -1.421875,
      "logps/chosen": -540.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0008544921875,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000827789306640625,
      "loss": 0.3476,
      "rewards/accuracies": 0.8799999952316284,
      "rewards/chosen": -1.6640625,
      "rewards/margins": 1.3046875,
      "rewards/rejected": -2.96875,
      "step": 5010
    },
    {
      "epoch": 9.85279685966634,
      "grad_norm": 12.015746876718154,
      "learning_rate": 2.880062091937252e-10,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.3984375,
      "logps/chosen": -520.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000904083251953125,
      "logps/rejected": -580.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.3555,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.6484375,
      "rewards/margins": 1.25,
      "rewards/rejected": -2.90625,
      "step": 5020
    },
    {
      "epoch": 9.872423945044162,
      "grad_norm": 13.922601928935721,
      "learning_rate": 2.1160717998913724e-10,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.40625,
      "logps/chosen": -580.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.00091552734375,
      "logps/rejected": -692.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000911712646484375,
      "loss": 0.3404,
      "rewards/accuracies": 0.8849999308586121,
      "rewards/chosen": -1.7421875,
      "rewards/margins": 1.34375,
      "rewards/rejected": -3.09375,
      "step": 5030
    },
    {
      "epoch": 9.892051030421982,
      "grad_norm": 11.484288890209422,
      "learning_rate": 1.4695576600948223e-10,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.4453125,
      "logps/chosen": -564.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.0008697509765625,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.000865936279296875,
      "loss": 0.3493,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.578125,
      "rewards/margins": 1.4140625,
      "rewards/rejected": -3.0,
      "step": 5040
    },
    {
      "epoch": 9.911678115799804,
      "grad_norm": 14.02364411561864,
      "learning_rate": 9.405500782808107e-11,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -568.0,
      "logps/chosen_bottom_tokens": -13.8125,
      "logps/chosen_top_tokens": -0.000911712646484375,
      "logps/rejected": -636.0,
      "logps/rejected_bottom_tokens": -13.6875,
      "logps/rejected_top_tokens": -0.00087738037109375,
      "loss": 0.358,
      "rewards/accuracies": 0.8950001001358032,
      "rewards/chosen": -1.625,
      "rewards/margins": 1.453125,
      "rewards/rejected": -3.078125,
      "step": 5050
    },
    {
      "epoch": 9.931305201177626,
      "grad_norm": 11.674058347699651,
      "learning_rate": 5.29073933818458e-11,
      "logits/chosen": -1.40625,
      "logits/rejected": -1.421875,
      "logps/chosen": -544.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000911712646484375,
      "logps/rejected": -660.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.000926971435546875,
      "loss": 0.3439,
      "rewards/accuracies": 0.8899999856948853,
      "rewards/chosen": -1.7265625,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -3.078125,
      "step": 5060
    },
    {
      "epoch": 9.950932286555446,
      "grad_norm": 12.556977231864163,
      "learning_rate": 2.351485785409557e-11,
      "logits/chosen": -1.3671875,
      "logits/rejected": -1.375,
      "logps/chosen": -532.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.000858306884765625,
      "logps/rejected": -644.0,
      "logps/rejected_bottom_tokens": -13.875,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.3606,
      "rewards/accuracies": 0.8650000691413879,
      "rewards/chosen": -1.5,
      "rewards/margins": 1.359375,
      "rewards/rejected": -2.859375,
      "step": 5070
    },
    {
      "epoch": 9.970559371933268,
      "grad_norm": 11.972937841080586,
      "learning_rate": 5.878783583712632e-12,
      "logits/chosen": -1.390625,
      "logits/rejected": -1.4296875,
      "logps/chosen": -502.0,
      "logps/chosen_bottom_tokens": -13.875,
      "logps/chosen_top_tokens": -0.001068115234375,
      "logps/rejected": -592.0,
      "logps/rejected_bottom_tokens": -13.75,
      "logps/rejected_top_tokens": -0.00106048583984375,
      "loss": 0.3563,
      "rewards/accuracies": 0.8449999690055847,
      "rewards/chosen": -1.6171875,
      "rewards/margins": 1.21875,
      "rewards/rejected": -2.84375,
      "step": 5080
    },
    {
      "epoch": 9.99018645731109,
      "grad_norm": 12.822609342422131,
      "learning_rate": 0.0,
      "logits/chosen": -1.3828125,
      "logits/rejected": -1.4140625,
      "logps/chosen": -536.0,
      "logps/chosen_bottom_tokens": -13.9375,
      "logps/chosen_top_tokens": -0.000896453857421875,
      "logps/rejected": -620.0,
      "logps/rejected_bottom_tokens": -13.8125,
      "logps/rejected_top_tokens": -0.0009002685546875,
      "loss": 0.3528,
      "rewards/accuracies": 0.8600000143051147,
      "rewards/chosen": -1.65625,
      "rewards/margins": 1.296875,
      "rewards/rejected": -2.953125,
      "step": 5090
    },
    {
      "epoch": 9.99018645731109,
      "step": 5090,
      "total_flos": 0.0,
      "train_loss": 0.40154263242522953,
      "train_runtime": 75277.8699,
      "train_samples_per_second": 8.121,
      "train_steps_per_second": 0.068
    }
  ],
  "logging_steps": 10,
  "max_steps": 5090,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 10,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 5,
  "trial_name": null,
  "trial_params": null
}