{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 1563,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 53.88663101196289,
      "kl": 0.023513054475188255,
      "learning_rate": 1.592356687898089e-08,
      "logps/chosen": -297.60601806640625,
      "logps/rejected": -325.4705810546875,
      "loss": 0.5832,
      "rewards/chosen": 0.0006075352430343628,
      "rewards/margins": -0.00015557828010059893,
      "rewards/rejected": 0.0007631134940311313,
      "step": 5
    },
    {
      "epoch": 0.01,
      "grad_norm": 50.249412536621094,
      "kl": 0.045294154435396194,
      "learning_rate": 3.184713375796178e-08,
      "logps/chosen": -287.09796142578125,
      "logps/rejected": -340.85723876953125,
      "loss": 0.584,
      "rewards/chosen": 0.01942053996026516,
      "rewards/margins": 0.016482431441545486,
      "rewards/rejected": 0.002938109915703535,
      "step": 10
    },
    {
      "epoch": 0.01,
      "grad_norm": 53.71184158325195,
      "kl": 0.3499985337257385,
      "learning_rate": 4.777070063694268e-08,
      "logps/chosen": -319.9255676269531,
      "logps/rejected": -329.686767578125,
      "loss": 0.5731,
      "rewards/chosen": 0.08261503279209137,
      "rewards/margins": 0.06412206590175629,
      "rewards/rejected": 0.01849297434091568,
      "step": 15
    },
    {
      "epoch": 0.01,
      "grad_norm": 54.06269073486328,
      "kl": 0.7365144491195679,
      "learning_rate": 6.369426751592356e-08,
      "logps/chosen": -303.22174072265625,
      "logps/rejected": -343.46942138671875,
      "loss": 0.5572,
      "rewards/chosen": 0.16511321067810059,
      "rewards/margins": 0.12509918212890625,
      "rewards/rejected": 0.04001404717564583,
      "step": 20
    },
    {
      "epoch": 0.02,
      "grad_norm": 51.1727294921875,
      "kl": 0.8321301341056824,
      "learning_rate": 7.961783439490445e-08,
      "logps/chosen": -319.74725341796875,
      "logps/rejected": -343.55291748046875,
      "loss": 0.5462,
      "rewards/chosen": 0.28225231170654297,
      "rewards/margins": 0.2485344409942627,
      "rewards/rejected": 0.033717863261699677,
      "step": 25
    },
    {
      "epoch": 0.02,
      "grad_norm": 51.29603576660156,
      "kl": 0.5910934209823608,
      "learning_rate": 9.554140127388536e-08,
      "logps/chosen": -284.5264892578125,
      "logps/rejected": -357.3599853515625,
      "loss": 0.5268,
      "rewards/chosen": 0.3866623044013977,
      "rewards/margins": 0.3753780722618103,
      "rewards/rejected": 0.01128421537578106,
      "step": 30
    },
    {
      "epoch": 0.02,
      "grad_norm": 49.49940872192383,
      "kl": 0.014250552281737328,
      "learning_rate": 1.1146496815286624e-07,
      "logps/chosen": -280.2434997558594,
      "logps/rejected": -375.0678405761719,
      "loss": 0.4927,
      "rewards/chosen": 0.38746222853660583,
      "rewards/margins": 0.5707041025161743,
      "rewards/rejected": -0.18324188888072968,
      "step": 35
    },
    {
      "epoch": 0.03,
      "grad_norm": 43.36540222167969,
      "kl": 0.0,
      "learning_rate": 1.2738853503184713e-07,
      "logps/chosen": -274.5870666503906,
      "logps/rejected": -330.31182861328125,
      "loss": 0.4715,
      "rewards/chosen": 0.33990636467933655,
      "rewards/margins": 0.7924365997314453,
      "rewards/rejected": -0.45253023505210876,
      "step": 40
    },
    {
      "epoch": 0.03,
      "grad_norm": 45.98810577392578,
      "kl": 0.0,
      "learning_rate": 1.43312101910828e-07,
      "logps/chosen": -304.57012939453125,
      "logps/rejected": -337.6307373046875,
      "loss": 0.4331,
      "rewards/chosen": 0.35226964950561523,
      "rewards/margins": 1.2349388599395752,
      "rewards/rejected": -0.8826690912246704,
      "step": 45
    },
    {
      "epoch": 0.03,
      "grad_norm": 47.690940856933594,
      "kl": 0.0,
      "learning_rate": 1.592356687898089e-07,
      "logps/chosen": -296.6683654785156,
      "logps/rejected": -354.68280029296875,
      "loss": 0.3929,
      "rewards/chosen": 0.3493489623069763,
      "rewards/margins": 1.7278648614883423,
      "rewards/rejected": -1.3785159587860107,
      "step": 50
    },
    {
      "epoch": 0.04,
      "grad_norm": 44.06206130981445,
      "kl": 0.0,
      "learning_rate": 1.7515923566878978e-07,
      "logps/chosen": -313.95135498046875,
      "logps/rejected": -300.2484436035156,
      "loss": 0.3888,
      "rewards/chosen": 0.27411553263664246,
      "rewards/margins": 1.9409167766571045,
      "rewards/rejected": -1.66680109500885,
      "step": 55
    },
    {
      "epoch": 0.04,
      "grad_norm": 41.619537353515625,
      "kl": 0.0,
      "learning_rate": 1.9108280254777072e-07,
      "logps/chosen": -295.02264404296875,
      "logps/rejected": -348.7203674316406,
      "loss": 0.3558,
      "rewards/chosen": 0.3618777394294739,
      "rewards/margins": 2.562573194503784,
      "rewards/rejected": -2.200695514678955,
      "step": 60
    },
    {
      "epoch": 0.04,
      "grad_norm": 38.81331253051758,
      "kl": 0.0,
      "learning_rate": 2.070063694267516e-07,
      "logps/chosen": -229.79873657226562,
      "logps/rejected": -352.5125732421875,
      "loss": 0.362,
      "rewards/chosen": 0.35671067237854004,
      "rewards/margins": 2.826559543609619,
      "rewards/rejected": -2.469848871231079,
      "step": 65
    },
    {
      "epoch": 0.04,
      "grad_norm": 46.18341827392578,
      "kl": 0.0,
      "learning_rate": 2.2292993630573247e-07,
      "logps/chosen": -297.68218994140625,
      "logps/rejected": -379.452880859375,
      "loss": 0.317,
      "rewards/chosen": 0.41898855566978455,
      "rewards/margins": 3.3871779441833496,
      "rewards/rejected": -2.968189239501953,
      "step": 70
    },
    {
      "epoch": 0.05,
      "grad_norm": 42.065818786621094,
      "kl": 0.0,
      "learning_rate": 2.388535031847134e-07,
      "logps/chosen": -287.422607421875,
      "logps/rejected": -360.3113708496094,
      "loss": 0.3633,
      "rewards/chosen": 0.4496755599975586,
      "rewards/margins": 3.4658408164978027,
      "rewards/rejected": -3.016165256500244,
      "step": 75
    },
    {
      "epoch": 0.05,
      "grad_norm": 42.938968658447266,
      "kl": 0.0,
      "learning_rate": 2.5477707006369425e-07,
      "logps/chosen": -315.4157409667969,
      "logps/rejected": -347.26983642578125,
      "loss": 0.3146,
      "rewards/chosen": 0.49002504348754883,
      "rewards/margins": 3.6589598655700684,
      "rewards/rejected": -3.1689350605010986,
      "step": 80
    },
    {
      "epoch": 0.05,
      "grad_norm": 35.48064422607422,
      "kl": 0.0,
      "learning_rate": 2.7070063694267513e-07,
      "logps/chosen": -265.584228515625,
      "logps/rejected": -382.82342529296875,
      "loss": 0.3244,
      "rewards/chosen": 0.5259283781051636,
      "rewards/margins": 4.123469352722168,
      "rewards/rejected": -3.5975403785705566,
      "step": 85
    },
    {
      "epoch": 0.06,
      "grad_norm": 40.673316955566406,
      "kl": 0.0,
      "learning_rate": 2.86624203821656e-07,
      "logps/chosen": -269.2232666015625,
      "logps/rejected": -394.211669921875,
      "loss": 0.3456,
      "rewards/chosen": 0.44943737983703613,
      "rewards/margins": 3.7313075065612793,
      "rewards/rejected": -3.2818703651428223,
      "step": 90
    },
    {
      "epoch": 0.06,
      "grad_norm": 43.21498489379883,
      "kl": 0.0,
      "learning_rate": 3.0254777070063694e-07,
      "logps/chosen": -259.74285888671875,
      "logps/rejected": -350.27001953125,
      "loss": 0.3277,
      "rewards/chosen": 0.4997062683105469,
      "rewards/margins": 3.3499958515167236,
      "rewards/rejected": -2.8502893447875977,
      "step": 95
    },
    {
      "epoch": 0.06,
      "grad_norm": 42.415992736816406,
      "kl": 0.0,
      "learning_rate": 3.184713375796178e-07,
      "logps/chosen": -285.27569580078125,
      "logps/rejected": -372.94757080078125,
      "loss": 0.3198,
      "rewards/chosen": 0.5120081901550293,
      "rewards/margins": 4.236746788024902,
      "rewards/rejected": -3.724738597869873,
      "step": 100
    },
    {
      "epoch": 0.07,
      "grad_norm": 41.03474044799805,
      "kl": 0.0,
      "learning_rate": 3.343949044585987e-07,
      "logps/chosen": -299.6357116699219,
      "logps/rejected": -353.11334228515625,
      "loss": 0.3132,
      "rewards/chosen": 0.4896661341190338,
      "rewards/margins": 4.393103122711182,
      "rewards/rejected": -3.9034361839294434,
      "step": 105
    },
    {
      "epoch": 0.07,
      "grad_norm": 37.77847671508789,
      "kl": 0.0,
      "learning_rate": 3.5031847133757957e-07,
      "logps/chosen": -239.7168731689453,
      "logps/rejected": -370.69866943359375,
      "loss": 0.3157,
      "rewards/chosen": 0.4915805459022522,
      "rewards/margins": 4.615452766418457,
      "rewards/rejected": -4.12387228012085,
      "step": 110
    },
    {
      "epoch": 0.07,
      "grad_norm": 44.40388488769531,
      "kl": 0.0,
      "learning_rate": 3.6624203821656045e-07,
      "logps/chosen": -292.5115051269531,
      "logps/rejected": -374.26849365234375,
      "loss": 0.3164,
      "rewards/chosen": 0.5263981819152832,
      "rewards/margins": 4.860444068908691,
      "rewards/rejected": -4.33404541015625,
      "step": 115
    },
    {
      "epoch": 0.08,
      "grad_norm": 39.663360595703125,
      "kl": 0.0,
      "learning_rate": 3.8216560509554143e-07,
      "logps/chosen": -305.62591552734375,
      "logps/rejected": -418.4881286621094,
      "loss": 0.2919,
      "rewards/chosen": 0.5590957999229431,
      "rewards/margins": 5.1840081214904785,
      "rewards/rejected": -4.624912261962891,
      "step": 120
    },
    {
      "epoch": 0.08,
      "grad_norm": 49.241607666015625,
      "kl": 0.0,
      "learning_rate": 3.980891719745223e-07,
      "logps/chosen": -299.54486083984375,
      "logps/rejected": -396.44586181640625,
      "loss": 0.302,
      "rewards/chosen": 0.5733675360679626,
      "rewards/margins": 5.690032482147217,
      "rewards/rejected": -5.116665363311768,
      "step": 125
    },
    {
      "epoch": 0.08,
      "grad_norm": 41.309715270996094,
      "kl": 0.0,
      "learning_rate": 4.140127388535032e-07,
      "logps/chosen": -285.24725341796875,
      "logps/rejected": -408.84326171875,
      "loss": 0.2895,
      "rewards/chosen": 0.5746848583221436,
      "rewards/margins": 5.906516075134277,
      "rewards/rejected": -5.331830978393555,
      "step": 130
    },
    {
      "epoch": 0.09,
      "grad_norm": 32.96284103393555,
      "kl": 0.0,
      "learning_rate": 4.2993630573248406e-07,
      "logps/chosen": -258.3543395996094,
      "logps/rejected": -388.1202087402344,
      "loss": 0.3022,
      "rewards/chosen": 0.5746804475784302,
      "rewards/margins": 5.607410430908203,
      "rewards/rejected": -5.0327301025390625,
      "step": 135
    },
    {
      "epoch": 0.09,
      "grad_norm": 45.53158187866211,
      "kl": 0.0,
      "learning_rate": 4.4585987261146494e-07,
      "logps/chosen": -292.7215270996094,
      "logps/rejected": -364.63116455078125,
      "loss": 0.2933,
      "rewards/chosen": 0.6362745761871338,
      "rewards/margins": 5.103418350219727,
      "rewards/rejected": -4.467143535614014,
      "step": 140
    },
    {
      "epoch": 0.09,
      "grad_norm": 33.09851837158203,
      "kl": 0.0,
      "learning_rate": 4.6178343949044587e-07,
      "logps/chosen": -287.439208984375,
      "logps/rejected": -362.47430419921875,
      "loss": 0.2741,
      "rewards/chosen": 0.5601326823234558,
      "rewards/margins": 5.778112888336182,
      "rewards/rejected": -5.217979431152344,
      "step": 145
    },
    {
      "epoch": 0.1,
      "grad_norm": 36.87670135498047,
      "kl": 0.0,
      "learning_rate": 4.777070063694267e-07,
      "logps/chosen": -275.13983154296875,
      "logps/rejected": -419.91290283203125,
      "loss": 0.3079,
      "rewards/chosen": 0.677828311920166,
      "rewards/margins": 5.985136985778809,
      "rewards/rejected": -5.307309150695801,
      "step": 150
    },
    {
      "epoch": 0.1,
      "grad_norm": 38.875118255615234,
      "kl": 0.0,
      "learning_rate": 4.936305732484076e-07,
      "logps/chosen": -301.48345947265625,
      "logps/rejected": -402.216796875,
      "loss": 0.2858,
      "rewards/chosen": 0.6449128985404968,
      "rewards/margins": 6.0093674659729,
      "rewards/rejected": -5.364454746246338,
      "step": 155
    },
    {
      "epoch": 0.1,
      "grad_norm": 34.41432189941406,
      "kl": 0.0,
      "learning_rate": 4.989331436699858e-07,
      "logps/chosen": -266.0882873535156,
      "logps/rejected": -377.15911865234375,
      "loss": 0.2722,
      "rewards/chosen": 0.7968356013298035,
      "rewards/margins": 6.45413064956665,
      "rewards/rejected": -5.657294750213623,
      "step": 160
    },
    {
      "epoch": 0.11,
      "grad_norm": 40.4543342590332,
      "kl": 0.0,
      "learning_rate": 4.971550497866287e-07,
      "logps/chosen": -310.2311096191406,
      "logps/rejected": -341.25482177734375,
      "loss": 0.2852,
      "rewards/chosen": 0.6570959091186523,
      "rewards/margins": 6.051398277282715,
      "rewards/rejected": -5.394301414489746,
      "step": 165
    },
    {
      "epoch": 0.11,
      "grad_norm": 42.1318359375,
      "kl": 0.0,
      "learning_rate": 4.953769559032717e-07,
      "logps/chosen": -270.304931640625,
      "logps/rejected": -369.9417724609375,
      "loss": 0.2895,
      "rewards/chosen": 0.6669288873672485,
      "rewards/margins": 6.19234561920166,
      "rewards/rejected": -5.525416374206543,
      "step": 170
    },
    {
      "epoch": 0.11,
      "grad_norm": 29.986738204956055,
      "kl": 0.0,
      "learning_rate": 4.935988620199146e-07,
      "logps/chosen": -276.00048828125,
      "logps/rejected": -409.60626220703125,
      "loss": 0.2606,
      "rewards/chosen": 0.7553092837333679,
      "rewards/margins": 6.95382022857666,
      "rewards/rejected": -6.198510646820068,
      "step": 175
    },
    {
      "epoch": 0.12,
      "grad_norm": 36.70397186279297,
      "kl": 0.0,
      "learning_rate": 4.918207681365576e-07,
      "logps/chosen": -275.0423583984375,
      "logps/rejected": -362.8970642089844,
      "loss": 0.2695,
      "rewards/chosen": 0.7985103726387024,
      "rewards/margins": 6.52749490737915,
      "rewards/rejected": -5.7289838790893555,
      "step": 180
    },
    {
      "epoch": 0.12,
      "grad_norm": 38.225425720214844,
      "kl": 0.0,
      "learning_rate": 4.900426742532006e-07,
      "logps/chosen": -310.0399475097656,
      "logps/rejected": -392.6236572265625,
      "loss": 0.2647,
      "rewards/chosen": 0.8835963010787964,
      "rewards/margins": 7.008363246917725,
      "rewards/rejected": -6.124766826629639,
      "step": 185
    },
    {
      "epoch": 0.12,
      "grad_norm": 37.8361930847168,
      "kl": 0.0,
      "learning_rate": 4.882645803698435e-07,
      "logps/chosen": -289.03814697265625,
      "logps/rejected": -387.9421081542969,
      "loss": 0.3058,
      "rewards/chosen": 0.6273610591888428,
      "rewards/margins": 6.493137359619141,
      "rewards/rejected": -5.865776538848877,
      "step": 190
    },
    {
      "epoch": 0.12,
      "grad_norm": 39.47262191772461,
      "kl": 0.0,
      "learning_rate": 4.864864864864865e-07,
      "logps/chosen": -316.1293029785156,
      "logps/rejected": -404.95843505859375,
      "loss": 0.2767,
      "rewards/chosen": 0.8789302110671997,
      "rewards/margins": 7.016871452331543,
      "rewards/rejected": -6.137940883636475,
      "step": 195
    },
    {
      "epoch": 0.13,
      "grad_norm": 33.78382110595703,
      "kl": 0.0,
      "learning_rate": 4.847083926031294e-07,
      "logps/chosen": -279.5328369140625,
      "logps/rejected": -444.06768798828125,
      "loss": 0.2948,
      "rewards/chosen": 0.8043686747550964,
      "rewards/margins": 7.215568542480469,
      "rewards/rejected": -6.411200523376465,
      "step": 200
    },
    {
      "epoch": 0.13,
      "grad_norm": 35.95137405395508,
      "kl": 0.0,
      "learning_rate": 4.829302987197724e-07,
      "logps/chosen": -314.162109375,
      "logps/rejected": -412.83203125,
      "loss": 0.2587,
      "rewards/chosen": 0.9799895286560059,
      "rewards/margins": 7.341480255126953,
      "rewards/rejected": -6.361490249633789,
      "step": 205
    },
    {
      "epoch": 0.13,
      "grad_norm": 28.62522315979004,
      "kl": 0.0,
      "learning_rate": 4.811522048364154e-07,
      "logps/chosen": -311.1414489746094,
      "logps/rejected": -400.20025634765625,
      "loss": 0.2843,
      "rewards/chosen": 0.4934808611869812,
      "rewards/margins": 6.236133098602295,
      "rewards/rejected": -5.742652416229248,
      "step": 210
    },
    {
      "epoch": 0.14,
      "grad_norm": 36.00955581665039,
      "kl": 0.0,
      "learning_rate": 4.793741109530583e-07,
      "logps/chosen": -308.5085754394531,
      "logps/rejected": -402.4859313964844,
      "loss": 0.2709,
      "rewards/chosen": 0.7078418135643005,
      "rewards/margins": 6.969674587249756,
      "rewards/rejected": -6.261832237243652,
      "step": 215
    },
    {
      "epoch": 0.14,
      "grad_norm": 36.267826080322266,
      "kl": 0.0,
      "learning_rate": 4.775960170697012e-07,
      "logps/chosen": -288.0454406738281,
      "logps/rejected": -410.33843994140625,
      "loss": 0.2625,
      "rewards/chosen": 0.7268310785293579,
      "rewards/margins": 7.102780342102051,
      "rewards/rejected": -6.375949382781982,
      "step": 220
    },
    {
      "epoch": 0.14,
      "grad_norm": 43.02051544189453,
      "kl": 0.0,
      "learning_rate": 4.7581792318634425e-07,
      "logps/chosen": -293.15704345703125,
      "logps/rejected": -410.19415283203125,
      "loss": 0.2691,
      "rewards/chosen": 0.8471654653549194,
      "rewards/margins": 7.160550117492676,
      "rewards/rejected": -6.313384056091309,
      "step": 225
    },
    {
      "epoch": 0.15,
      "grad_norm": 35.434814453125,
      "kl": 0.0,
      "learning_rate": 4.7403982930298717e-07,
      "logps/chosen": -329.09716796875,
      "logps/rejected": -404.0379333496094,
      "loss": 0.2705,
      "rewards/chosen": 0.8330980539321899,
      "rewards/margins": 7.071954250335693,
      "rewards/rejected": -6.238855838775635,
      "step": 230
    },
    {
      "epoch": 0.15,
      "grad_norm": 36.30598068237305,
      "kl": 0.0,
      "learning_rate": 4.7226173541963014e-07,
      "logps/chosen": -251.01351928710938,
      "logps/rejected": -404.2989196777344,
      "loss": 0.2729,
      "rewards/chosen": 0.7619932293891907,
      "rewards/margins": 6.793028354644775,
      "rewards/rejected": -6.03103494644165,
      "step": 235
    },
    {
      "epoch": 0.15,
      "grad_norm": 46.27096939086914,
      "kl": 0.0,
      "learning_rate": 4.7048364153627306e-07,
      "logps/chosen": -286.8477478027344,
      "logps/rejected": -389.7752685546875,
      "loss": 0.2742,
      "rewards/chosen": 0.79796302318573,
      "rewards/margins": 7.297283172607422,
      "rewards/rejected": -6.499319553375244,
      "step": 240
    },
    {
      "epoch": 0.16,
      "grad_norm": 45.02741241455078,
      "kl": 0.0,
      "learning_rate": 4.6870554765291604e-07,
      "logps/chosen": -308.306396484375,
      "logps/rejected": -399.7406311035156,
      "loss": 0.2764,
      "rewards/chosen": 0.8131675720214844,
      "rewards/margins": 7.1386399269104,
      "rewards/rejected": -6.325472354888916,
      "step": 245
    },
    {
      "epoch": 0.16,
      "grad_norm": 35.437339782714844,
      "kl": 0.0,
      "learning_rate": 4.66927453769559e-07,
      "logps/chosen": -259.05401611328125,
      "logps/rejected": -398.1236877441406,
      "loss": 0.2899,
      "rewards/chosen": 0.6805434823036194,
      "rewards/margins": 7.213263511657715,
      "rewards/rejected": -6.532719612121582,
      "step": 250
    },
    {
      "epoch": 0.16,
      "grad_norm": 35.13607406616211,
      "kl": 0.0,
      "learning_rate": 4.65149359886202e-07,
      "logps/chosen": -256.3832092285156,
      "logps/rejected": -410.2801208496094,
      "loss": 0.2647,
      "rewards/chosen": 0.9173786044120789,
      "rewards/margins": 7.905294895172119,
      "rewards/rejected": -6.987915992736816,
      "step": 255
    },
    {
      "epoch": 0.17,
      "grad_norm": 35.4403190612793,
      "kl": 0.0,
      "learning_rate": 4.633712660028449e-07,
      "logps/chosen": -318.9344787597656,
      "logps/rejected": -403.5733337402344,
      "loss": 0.2669,
      "rewards/chosen": 0.9034342765808105,
      "rewards/margins": 7.315404415130615,
      "rewards/rejected": -6.411970615386963,
      "step": 260
    },
    {
      "epoch": 0.17,
      "grad_norm": 40.87915802001953,
      "kl": 0.0,
      "learning_rate": 4.615931721194879e-07,
      "logps/chosen": -307.8759765625,
      "logps/rejected": -405.61383056640625,
      "loss": 0.2424,
      "rewards/chosen": 0.9530359506607056,
      "rewards/margins": 8.030720710754395,
      "rewards/rejected": -7.0776848793029785,
      "step": 265
    },
    {
      "epoch": 0.17,
      "grad_norm": 37.06570816040039,
      "kl": 0.0,
      "learning_rate": 4.5981507823613085e-07,
      "logps/chosen": -279.9657897949219,
      "logps/rejected": -429.54638671875,
      "loss": 0.2654,
      "rewards/chosen": 1.0000529289245605,
      "rewards/margins": 7.914543151855469,
      "rewards/rejected": -6.91448974609375,
      "step": 270
    },
    {
      "epoch": 0.18,
      "grad_norm": 35.289634704589844,
      "kl": 0.0,
      "learning_rate": 4.580369843527738e-07,
      "logps/chosen": -314.99267578125,
      "logps/rejected": -417.67401123046875,
      "loss": 0.2673,
      "rewards/chosen": 0.7197456955909729,
      "rewards/margins": 7.778171539306641,
      "rewards/rejected": -7.0584259033203125,
      "step": 275
    },
    {
      "epoch": 0.18,
      "grad_norm": 34.34975814819336,
      "kl": 0.0,
      "learning_rate": 4.562588904694168e-07,
      "logps/chosen": -266.03350830078125,
      "logps/rejected": -420.0655822753906,
      "loss": 0.2683,
      "rewards/chosen": 0.8104265332221985,
      "rewards/margins": 7.52780294418335,
      "rewards/rejected": -6.717376708984375,
      "step": 280
    },
    {
      "epoch": 0.18,
      "grad_norm": 36.18477249145508,
      "kl": 0.0,
      "learning_rate": 4.544807965860597e-07,
      "logps/chosen": -312.69708251953125,
      "logps/rejected": -419.86883544921875,
      "loss": 0.2483,
      "rewards/chosen": 0.8221893310546875,
      "rewards/margins": 7.910370826721191,
      "rewards/rejected": -7.088181972503662,
      "step": 285
    },
    {
      "epoch": 0.19,
      "grad_norm": 34.63364028930664,
      "kl": 0.0,
      "learning_rate": 4.5270270270270264e-07,
      "logps/chosen": -286.63555908203125,
      "logps/rejected": -437.01031494140625,
      "loss": 0.2576,
      "rewards/chosen": 1.0530911684036255,
      "rewards/margins": 8.051950454711914,
      "rewards/rejected": -6.998859405517578,
      "step": 290
    },
    {
      "epoch": 0.19,
      "grad_norm": 33.81501388549805,
      "kl": 0.0,
      "learning_rate": 4.509246088193456e-07,
      "logps/chosen": -295.6054992675781,
      "logps/rejected": -395.41796875,
      "loss": 0.2539,
      "rewards/chosen": 1.0546354055404663,
      "rewards/margins": 7.8214921951293945,
      "rewards/rejected": -6.766855716705322,
      "step": 295
    },
    {
      "epoch": 0.19,
      "grad_norm": 38.72855758666992,
      "kl": 0.0,
      "learning_rate": 4.491465149359886e-07,
      "logps/chosen": -305.07586669921875,
      "logps/rejected": -399.9986267089844,
      "loss": 0.2502,
      "rewards/chosen": 1.029415488243103,
      "rewards/margins": 8.159204483032227,
      "rewards/rejected": -7.129788398742676,
      "step": 300
    },
    {
      "epoch": 0.2,
      "grad_norm": 36.09114456176758,
      "kl": 0.0,
      "learning_rate": 4.4736842105263156e-07,
      "logps/chosen": -252.03683471679688,
      "logps/rejected": -418.17938232421875,
      "loss": 0.2554,
      "rewards/chosen": 0.9251939058303833,
      "rewards/margins": 8.25263786315918,
      "rewards/rejected": -7.327445030212402,
      "step": 305
    },
    {
      "epoch": 0.2,
      "grad_norm": 39.85062026977539,
      "kl": 0.0,
      "learning_rate": 4.4559032716927454e-07,
      "logps/chosen": -311.30535888671875,
      "logps/rejected": -386.61248779296875,
      "loss": 0.2836,
      "rewards/chosen": 0.8921693563461304,
      "rewards/margins": 7.110099792480469,
      "rewards/rejected": -6.217930793762207,
      "step": 310
    },
    {
      "epoch": 0.2,
      "grad_norm": 31.65120506286621,
      "kl": 0.0,
      "learning_rate": 4.438122332859175e-07,
      "logps/chosen": -270.47064208984375,
      "logps/rejected": -450.70965576171875,
      "loss": 0.2624,
      "rewards/chosen": 0.9781096577644348,
      "rewards/margins": 8.819513320922852,
      "rewards/rejected": -7.841403007507324,
      "step": 315
    },
    {
      "epoch": 0.2,
      "grad_norm": 43.421443939208984,
      "kl": 0.0,
      "learning_rate": 4.420341394025605e-07,
      "logps/chosen": -290.4539489746094,
      "logps/rejected": -405.73968505859375,
      "loss": 0.2676,
      "rewards/chosen": 0.9018673896789551,
      "rewards/margins": 8.168411254882812,
      "rewards/rejected": -7.266544342041016,
      "step": 320
    },
    {
      "epoch": 0.21,
      "grad_norm": 36.024681091308594,
      "kl": 0.0,
      "learning_rate": 4.4025604551920335e-07,
      "logps/chosen": -278.21484375,
      "logps/rejected": -393.1819152832031,
      "loss": 0.2447,
      "rewards/chosen": 0.9998067617416382,
      "rewards/margins": 7.97442626953125,
      "rewards/rejected": -6.9746198654174805,
      "step": 325
    },
    {
      "epoch": 0.21,
      "grad_norm": 31.11004066467285,
      "kl": 0.0,
      "learning_rate": 4.384779516358463e-07,
      "logps/chosen": -249.5725555419922,
      "logps/rejected": -443.29388427734375,
      "loss": 0.2397,
      "rewards/chosen": 0.7795785665512085,
      "rewards/margins": 8.376214981079102,
      "rewards/rejected": -7.596636772155762,
      "step": 330
    },
    {
      "epoch": 0.21,
      "grad_norm": 36.15337371826172,
      "kl": 0.0,
      "learning_rate": 4.366998577524893e-07,
      "logps/chosen": -283.8440856933594,
      "logps/rejected": -403.4510498046875,
      "loss": 0.2555,
      "rewards/chosen": 0.98023521900177,
      "rewards/margins": 8.079346656799316,
      "rewards/rejected": -7.099111080169678,
      "step": 335
    },
    {
      "epoch": 0.22,
      "grad_norm": 35.31691360473633,
      "kl": 0.0,
      "learning_rate": 4.3492176386913227e-07,
      "logps/chosen": -265.3778381347656,
      "logps/rejected": -388.70953369140625,
      "loss": 0.2543,
      "rewards/chosen": 0.9121273159980774,
      "rewards/margins": 7.6136298179626465,
      "rewards/rejected": -6.701502799987793,
      "step": 340
    },
    {
      "epoch": 0.22,
      "grad_norm": 29.128305435180664,
      "kl": 0.0,
      "learning_rate": 4.3314366998577524e-07,
      "logps/chosen": -257.42572021484375,
      "logps/rejected": -397.4496765136719,
      "loss": 0.2452,
      "rewards/chosen": 1.0432583093643188,
      "rewards/margins": 7.451505184173584,
      "rewards/rejected": -6.4082465171813965,
      "step": 345
    },
    {
      "epoch": 0.22,
      "grad_norm": 32.244834899902344,
      "kl": 0.0,
      "learning_rate": 4.313655761024182e-07,
      "logps/chosen": -290.0133056640625,
      "logps/rejected": -423.0615234375,
      "loss": 0.2449,
      "rewards/chosen": 1.174845576286316,
      "rewards/margins": 8.203340530395508,
      "rewards/rejected": -7.028494358062744,
      "step": 350
    },
    {
      "epoch": 0.23,
      "grad_norm": 37.363033294677734,
      "kl": 0.0,
      "learning_rate": 4.2958748221906114e-07,
      "logps/chosen": -325.817138671875,
      "logps/rejected": -419.47589111328125,
      "loss": 0.2419,
      "rewards/chosen": 1.2895926237106323,
      "rewards/margins": 8.572031021118164,
      "rewards/rejected": -7.282437801361084,
      "step": 355
    },
    {
      "epoch": 0.23,
      "grad_norm": 35.990299224853516,
      "kl": 0.0,
      "learning_rate": 4.278093883357041e-07,
      "logps/chosen": -259.43365478515625,
      "logps/rejected": -414.167724609375,
      "loss": 0.2343,
      "rewards/chosen": 1.0641326904296875,
      "rewards/margins": 8.207174301147461,
      "rewards/rejected": -7.143041133880615,
      "step": 360
    },
    {
      "epoch": 0.23,
      "grad_norm": 34.36444091796875,
      "kl": 0.0,
      "learning_rate": 4.260312944523471e-07,
      "logps/chosen": -293.53240966796875,
      "logps/rejected": -361.9354553222656,
      "loss": 0.2701,
      "rewards/chosen": 0.8572598695755005,
      "rewards/margins": 7.768780708312988,
      "rewards/rejected": -6.911520957946777,
      "step": 365
    },
    {
      "epoch": 0.24,
      "grad_norm": 36.629608154296875,
      "kl": 0.0,
      "learning_rate": 4.2425320056899e-07,
      "logps/chosen": -279.6189880371094,
      "logps/rejected": -408.66680908203125,
      "loss": 0.2506,
      "rewards/chosen": 0.9547025561332703,
      "rewards/margins": 8.172720909118652,
      "rewards/rejected": -7.218017578125,
      "step": 370
    },
    {
      "epoch": 0.24,
      "grad_norm": 38.02170944213867,
      "kl": 0.0,
      "learning_rate": 4.22475106685633e-07,
      "logps/chosen": -287.5861511230469,
      "logps/rejected": -396.6712341308594,
      "loss": 0.2446,
      "rewards/chosen": 1.0745197534561157,
      "rewards/margins": 7.910072326660156,
      "rewards/rejected": -6.835552215576172,
      "step": 375
    },
    {
      "epoch": 0.24,
      "grad_norm": 38.12566375732422,
      "kl": 0.0,
      "learning_rate": 4.2069701280227595e-07,
      "logps/chosen": -255.07809448242188,
      "logps/rejected": -413.735595703125,
      "loss": 0.2231,
      "rewards/chosen": 0.9518634676933289,
      "rewards/margins": 8.076669692993164,
      "rewards/rejected": -7.124806880950928,
      "step": 380
    },
    {
      "epoch": 0.25,
      "grad_norm": 34.425228118896484,
      "kl": 0.0,
      "learning_rate": 4.189189189189189e-07,
      "logps/chosen": -240.9487762451172,
      "logps/rejected": -432.3330993652344,
      "loss": 0.2357,
      "rewards/chosen": 1.2129076719284058,
      "rewards/margins": 8.845483779907227,
      "rewards/rejected": -7.632575035095215,
      "step": 385
    },
    {
      "epoch": 0.25,
      "grad_norm": 34.311222076416016,
      "kl": 0.0,
      "learning_rate": 4.1714082503556185e-07,
      "logps/chosen": -255.38522338867188,
      "logps/rejected": -384.539794921875,
      "loss": 0.2416,
      "rewards/chosen": 1.1686419248580933,
      "rewards/margins": 8.326786041259766,
      "rewards/rejected": -7.158143520355225,
      "step": 390
    },
    {
      "epoch": 0.25,
      "grad_norm": 33.39529800415039,
      "kl": 0.0,
      "learning_rate": 4.153627311522048e-07,
      "logps/chosen": -287.487060546875,
      "logps/rejected": -402.24749755859375,
      "loss": 0.2599,
      "rewards/chosen": 1.0036966800689697,
      "rewards/margins": 8.261211395263672,
      "rewards/rejected": -7.257514953613281,
      "step": 395
    },
    {
      "epoch": 0.26,
      "grad_norm": 31.273969650268555,
      "kl": 0.0,
      "learning_rate": 4.135846372688478e-07,
      "logps/chosen": -297.91522216796875,
      "logps/rejected": -443.12835693359375,
      "loss": 0.2628,
      "rewards/chosen": 1.006545066833496,
      "rewards/margins": 8.5576810836792,
      "rewards/rejected": -7.551136016845703,
      "step": 400
    },
    {
      "epoch": 0.26,
      "grad_norm": 28.262744903564453,
      "kl": 0.0,
      "learning_rate": 4.1180654338549077e-07,
      "logps/chosen": -328.5188903808594,
      "logps/rejected": -388.5363464355469,
      "loss": 0.2463,
      "rewards/chosen": 1.4216535091400146,
      "rewards/margins": 8.22613525390625,
      "rewards/rejected": -6.804482460021973,
      "step": 405
    },
    {
      "epoch": 0.26,
      "grad_norm": 31.14788246154785,
      "kl": 0.0,
      "learning_rate": 4.100284495021337e-07,
      "logps/chosen": -300.68963623046875,
      "logps/rejected": -445.842529296875,
      "loss": 0.2304,
      "rewards/chosen": 1.0163438320159912,
      "rewards/margins": 9.177450180053711,
      "rewards/rejected": -8.161107063293457,
      "step": 410
    },
    {
      "epoch": 0.27,
      "grad_norm": 27.50638771057129,
      "kl": 0.0,
      "learning_rate": 4.082503556187766e-07,
      "logps/chosen": -278.8860168457031,
      "logps/rejected": -449.13104248046875,
      "loss": 0.2188,
      "rewards/chosen": 1.2694514989852905,
      "rewards/margins": 9.333213806152344,
      "rewards/rejected": -8.063762664794922,
      "step": 415
    },
    {
      "epoch": 0.27,
      "grad_norm": 35.6218376159668,
      "kl": 0.0,
      "learning_rate": 4.064722617354196e-07,
      "logps/chosen": -280.0066833496094,
      "logps/rejected": -398.3285827636719,
      "loss": 0.2508,
      "rewards/chosen": 1.4511891603469849,
      "rewards/margins": 8.606059074401855,
      "rewards/rejected": -7.15487003326416,
      "step": 420
    },
    {
      "epoch": 0.27,
      "grad_norm": 40.70615768432617,
      "kl": 0.0,
      "learning_rate": 4.0469416785206256e-07,
      "logps/chosen": -331.06085205078125,
      "logps/rejected": -405.2574768066406,
      "loss": 0.2451,
      "rewards/chosen": 1.0914924144744873,
      "rewards/margins": 8.246747016906738,
      "rewards/rejected": -7.155255317687988,
      "step": 425
    },
    {
      "epoch": 0.28,
      "grad_norm": 36.7715950012207,
      "kl": 0.0,
      "learning_rate": 4.0291607396870553e-07,
      "logps/chosen": -257.7249450683594,
      "logps/rejected": -490.748046875,
      "loss": 0.2151,
      "rewards/chosen": 1.230444073677063,
      "rewards/margins": 9.618934631347656,
      "rewards/rejected": -8.388490676879883,
      "step": 430
    },
    {
      "epoch": 0.28,
      "grad_norm": 34.04132080078125,
      "kl": 0.0,
      "learning_rate": 4.011379800853485e-07,
      "logps/chosen": -270.65325927734375,
      "logps/rejected": -423.4205017089844,
      "loss": 0.2498,
      "rewards/chosen": 1.293565034866333,
      "rewards/margins": 8.601400375366211,
      "rewards/rejected": -7.307835578918457,
      "step": 435
    },
    {
      "epoch": 0.28,
      "grad_norm": 34.57319259643555,
      "kl": 0.0,
      "learning_rate": 3.993598862019915e-07,
      "logps/chosen": -298.8142395019531,
      "logps/rejected": -436.76202392578125,
      "loss": 0.2337,
      "rewards/chosen": 1.1631160974502563,
      "rewards/margins": 9.01208782196045,
      "rewards/rejected": -7.848972320556641,
      "step": 440
    },
    {
      "epoch": 0.28,
      "grad_norm": 39.89948272705078,
      "kl": 0.0,
      "learning_rate": 3.975817923186344e-07,
      "logps/chosen": -262.2899475097656,
      "logps/rejected": -423.69189453125,
      "loss": 0.2658,
      "rewards/chosen": 1.0408456325531006,
      "rewards/margins": 8.8524808883667,
      "rewards/rejected": -7.8116350173950195,
      "step": 445
    },
    {
      "epoch": 0.29,
      "grad_norm": 31.670787811279297,
      "kl": 0.0,
      "learning_rate": 3.9580369843527737e-07,
      "logps/chosen": -290.41326904296875,
      "logps/rejected": -415.8921813964844,
      "loss": 0.2251,
      "rewards/chosen": 1.409716010093689,
      "rewards/margins": 8.17490291595459,
      "rewards/rejected": -6.7651872634887695,
      "step": 450
    },
    {
      "epoch": 0.29,
      "grad_norm": 29.412275314331055,
      "kl": 0.0,
      "learning_rate": 3.940256045519203e-07,
      "logps/chosen": -270.35009765625,
      "logps/rejected": -436.0580139160156,
      "loss": 0.2476,
      "rewards/chosen": 1.0782283544540405,
      "rewards/margins": 8.212353706359863,
      "rewards/rejected": -7.134124755859375,
      "step": 455
    },
    {
      "epoch": 0.29,
      "grad_norm": 34.431488037109375,
      "kl": 0.0,
      "learning_rate": 3.9224751066856327e-07,
      "logps/chosen": -283.64166259765625,
      "logps/rejected": -411.4891662597656,
      "loss": 0.2534,
      "rewards/chosen": 1.0376536846160889,
      "rewards/margins": 7.781105041503906,
      "rewards/rejected": -6.743451118469238,
      "step": 460
    },
    {
      "epoch": 0.3,
      "grad_norm": 30.757057189941406,
      "kl": 0.0,
      "learning_rate": 3.9046941678520624e-07,
      "logps/chosen": -265.84423828125,
      "logps/rejected": -379.9093933105469,
      "loss": 0.2352,
      "rewards/chosen": 1.3857548236846924,
      "rewards/margins": 8.423876762390137,
      "rewards/rejected": -7.038122653961182,
      "step": 465
    },
    {
      "epoch": 0.3,
      "grad_norm": 31.575454711914062,
      "kl": 0.0,
      "learning_rate": 3.886913229018492e-07,
      "logps/chosen": -264.2037048339844,
      "logps/rejected": -408.8255920410156,
      "loss": 0.2324,
      "rewards/chosen": 1.3042352199554443,
      "rewards/margins": 8.68317985534668,
      "rewards/rejected": -7.378944396972656,
      "step": 470
    },
    {
      "epoch": 0.3,
      "grad_norm": 34.39158630371094,
      "kl": 0.0,
      "learning_rate": 3.8691322901849213e-07,
      "logps/chosen": -271.2469177246094,
      "logps/rejected": -422.51129150390625,
      "loss": 0.2041,
      "rewards/chosen": 1.7015053033828735,
      "rewards/margins": 9.282489776611328,
      "rewards/rejected": -7.580985069274902,
      "step": 475
    },
    {
      "epoch": 0.31,
      "grad_norm": 40.40760040283203,
      "kl": 0.0,
      "learning_rate": 3.851351351351351e-07,
      "logps/chosen": -271.46319580078125,
      "logps/rejected": -394.5502014160156,
      "loss": 0.2627,
      "rewards/chosen": 1.2340047359466553,
      "rewards/margins": 7.9939703941345215,
      "rewards/rejected": -6.7599663734436035,
      "step": 480
    },
    {
      "epoch": 0.31,
      "grad_norm": 38.451168060302734,
      "kl": 0.0,
      "learning_rate": 3.833570412517781e-07,
      "logps/chosen": -291.4540100097656,
      "logps/rejected": -382.9058532714844,
      "loss": 0.2501,
      "rewards/chosen": 1.120945930480957,
      "rewards/margins": 7.974093437194824,
      "rewards/rejected": -6.853147983551025,
      "step": 485
    },
    {
      "epoch": 0.31,
      "grad_norm": 36.719886779785156,
      "kl": 0.0,
      "learning_rate": 3.8157894736842105e-07,
      "logps/chosen": -314.1134338378906,
      "logps/rejected": -406.29437255859375,
      "loss": 0.2247,
      "rewards/chosen": 1.3398951292037964,
      "rewards/margins": 9.037193298339844,
      "rewards/rejected": -7.6972975730896,
      "step": 490
    },
    {
      "epoch": 0.32,
      "grad_norm": 31.631505966186523,
      "kl": 0.0,
      "learning_rate": 3.7980085348506403e-07,
      "logps/chosen": -290.2677307128906,
      "logps/rejected": -407.5957336425781,
      "loss": 0.2352,
      "rewards/chosen": 1.519084095954895,
      "rewards/margins": 8.60625171661377,
      "rewards/rejected": -7.087167263031006,
      "step": 495
    },
    {
      "epoch": 0.32,
      "grad_norm": 25.598243713378906,
      "kl": 0.0,
      "learning_rate": 3.7802275960170695e-07,
      "logps/chosen": -261.9007263183594,
      "logps/rejected": -433.05047607421875,
      "loss": 0.2367,
      "rewards/chosen": 1.5405170917510986,
      "rewards/margins": 9.024420738220215,
      "rewards/rejected": -7.483903408050537,
      "step": 500
    },
    {
      "epoch": 0.32,
      "grad_norm": 30.555438995361328,
      "kl": 0.0,
      "learning_rate": 3.7624466571834987e-07,
      "logps/chosen": -278.92108154296875,
      "logps/rejected": -417.1661682128906,
      "loss": 0.2369,
      "rewards/chosen": 1.382774829864502,
      "rewards/margins": 8.678990364074707,
      "rewards/rejected": -7.296216011047363,
      "step": 505
    },
    {
      "epoch": 0.33,
      "grad_norm": 25.65595245361328,
      "kl": 0.0,
      "learning_rate": 3.7446657183499284e-07,
      "logps/chosen": -259.3731384277344,
      "logps/rejected": -409.6930236816406,
      "loss": 0.2376,
      "rewards/chosen": 1.2507401704788208,
      "rewards/margins": 8.60232162475586,
      "rewards/rejected": -7.351581573486328,
      "step": 510
    },
    {
      "epoch": 0.33,
      "grad_norm": 36.59701156616211,
      "kl": 0.0,
      "learning_rate": 3.726884779516358e-07,
      "logps/chosen": -287.05059814453125,
      "logps/rejected": -396.98638916015625,
      "loss": 0.2075,
      "rewards/chosen": 1.687380075454712,
      "rewards/margins": 9.18794059753418,
      "rewards/rejected": -7.5005598068237305,
      "step": 515
    },
    {
      "epoch": 0.33,
      "grad_norm": 36.45357131958008,
      "kl": 0.0,
      "learning_rate": 3.709103840682788e-07,
      "logps/chosen": -267.5130920410156,
      "logps/rejected": -435.63885498046875,
      "loss": 0.2306,
      "rewards/chosen": 1.1616590023040771,
      "rewards/margins": 9.239825248718262,
      "rewards/rejected": -8.078166007995605,
      "step": 520
    },
    {
      "epoch": 0.34,
      "grad_norm": 36.515785217285156,
      "kl": 0.0,
      "learning_rate": 3.6913229018492176e-07,
      "logps/chosen": -281.87744140625,
      "logps/rejected": -413.88006591796875,
      "loss": 0.2496,
      "rewards/chosen": 1.3109562397003174,
      "rewards/margins": 8.770210266113281,
      "rewards/rejected": -7.459254264831543,
      "step": 525
    },
    {
      "epoch": 0.34,
      "grad_norm": 31.515460968017578,
      "kl": 0.0,
      "learning_rate": 3.6735419630156474e-07,
      "logps/chosen": -224.22750854492188,
      "logps/rejected": -424.96649169921875,
      "loss": 0.2045,
      "rewards/chosen": 1.2929831743240356,
      "rewards/margins": 8.969011306762695,
      "rewards/rejected": -7.676026821136475,
      "step": 530
    },
    {
      "epoch": 0.34,
      "grad_norm": 29.997665405273438,
      "kl": 0.0,
      "learning_rate": 3.655761024182077e-07,
      "logps/chosen": -289.0644836425781,
      "logps/rejected": -391.5130920410156,
      "loss": 0.252,
      "rewards/chosen": 1.3215197324752808,
      "rewards/margins": 9.083866119384766,
      "rewards/rejected": -7.7623467445373535,
      "step": 535
    },
    {
      "epoch": 0.35,
      "grad_norm": 33.1764030456543,
      "kl": 0.0,
      "learning_rate": 3.637980085348506e-07,
      "logps/chosen": -252.8762664794922,
      "logps/rejected": -432.19952392578125,
      "loss": 0.2093,
      "rewards/chosen": 1.495452880859375,
      "rewards/margins": 9.005461692810059,
      "rewards/rejected": -7.510009765625,
      "step": 540
    },
    {
      "epoch": 0.35,
      "grad_norm": 28.675111770629883,
      "kl": 0.0,
      "learning_rate": 3.6201991465149355e-07,
      "logps/chosen": -260.71624755859375,
      "logps/rejected": -408.84759521484375,
      "loss": 0.206,
      "rewards/chosen": 1.433053970336914,
      "rewards/margins": 8.826322555541992,
      "rewards/rejected": -7.393267631530762,
      "step": 545
    },
    {
      "epoch": 0.35,
      "grad_norm": 39.82828903198242,
      "kl": 0.0,
      "learning_rate": 3.602418207681365e-07,
      "logps/chosen": -365.56134033203125,
      "logps/rejected": -382.9657897949219,
      "loss": 0.2252,
      "rewards/chosen": 1.4427438974380493,
      "rewards/margins": 9.311979293823242,
      "rewards/rejected": -7.869235992431641,
      "step": 550
    },
    {
      "epoch": 0.36,
      "grad_norm": 29.757482528686523,
      "kl": 0.0,
      "learning_rate": 3.584637268847795e-07,
      "logps/chosen": -270.66815185546875,
      "logps/rejected": -418.73883056640625,
      "loss": 0.2445,
      "rewards/chosen": 1.2224515676498413,
      "rewards/margins": 9.511232376098633,
      "rewards/rejected": -8.28878116607666,
      "step": 555
    },
    {
      "epoch": 0.36,
      "grad_norm": 32.810874938964844,
      "kl": 0.0,
      "learning_rate": 3.5668563300142247e-07,
      "logps/chosen": -280.0058288574219,
      "logps/rejected": -459.1136779785156,
      "loss": 0.2115,
      "rewards/chosen": 1.7389733791351318,
      "rewards/margins": 9.870342254638672,
      "rewards/rejected": -8.131368637084961,
      "step": 560
    },
    {
      "epoch": 0.36,
      "grad_norm": 34.706581115722656,
      "kl": 0.0,
      "learning_rate": 3.5490753911806545e-07,
      "logps/chosen": -251.6792755126953,
      "logps/rejected": -390.3374328613281,
      "loss": 0.2354,
      "rewards/chosen": 1.290079116821289,
      "rewards/margins": 8.472095489501953,
      "rewards/rejected": -7.182016849517822,
      "step": 565
    },
    {
      "epoch": 0.36,
      "grad_norm": 37.13447952270508,
      "kl": 0.0,
      "learning_rate": 3.5312944523470837e-07,
      "logps/chosen": -291.58502197265625,
      "logps/rejected": -395.8794860839844,
      "loss": 0.2129,
      "rewards/chosen": 1.712994933128357,
      "rewards/margins": 9.457635879516602,
      "rewards/rejected": -7.744640350341797,
      "step": 570
    },
    {
      "epoch": 0.37,
      "grad_norm": 37.750213623046875,
      "kl": 0.0,
      "learning_rate": 3.5135135135135134e-07,
      "logps/chosen": -275.55645751953125,
      "logps/rejected": -421.70611572265625,
      "loss": 0.2109,
      "rewards/chosen": 1.398840308189392,
      "rewards/margins": 9.466266632080078,
      "rewards/rejected": -8.067425727844238,
      "step": 575
    },
    {
      "epoch": 0.37,
      "grad_norm": 29.82187843322754,
      "kl": 0.0,
      "learning_rate": 3.495732574679943e-07,
      "logps/chosen": -275.10003662109375,
      "logps/rejected": -414.6404724121094,
      "loss": 0.251,
      "rewards/chosen": 1.1742942333221436,
      "rewards/margins": 8.796428680419922,
      "rewards/rejected": -7.622133731842041,
      "step": 580
    },
    {
      "epoch": 0.37,
      "grad_norm": 34.12580490112305,
      "kl": 0.0,
      "learning_rate": 3.4779516358463724e-07,
      "logps/chosen": -273.7625427246094,
      "logps/rejected": -423.67364501953125,
      "loss": 0.2324,
      "rewards/chosen": 1.2637519836425781,
      "rewards/margins": 9.396784782409668,
      "rewards/rejected": -8.13303279876709,
      "step": 585
    },
    {
      "epoch": 0.38,
      "grad_norm": 32.073856353759766,
      "kl": 0.0,
      "learning_rate": 3.460170697012802e-07,
      "logps/chosen": -240.8434295654297,
      "logps/rejected": -415.4698791503906,
      "loss": 0.2355,
      "rewards/chosen": 1.2490732669830322,
      "rewards/margins": 8.936406135559082,
      "rewards/rejected": -7.6873321533203125,
      "step": 590
    },
    {
      "epoch": 0.38,
      "grad_norm": 33.48440170288086,
      "kl": 0.0,
      "learning_rate": 3.442389758179232e-07,
      "logps/chosen": -281.2816467285156,
      "logps/rejected": -413.6434631347656,
      "loss": 0.2315,
      "rewards/chosen": 1.444791555404663,
      "rewards/margins": 8.960540771484375,
      "rewards/rejected": -7.515749454498291,
      "step": 595
    },
    {
      "epoch": 0.38,
      "grad_norm": 36.15184020996094,
      "kl": 0.0,
      "learning_rate": 3.424608819345661e-07,
      "logps/chosen": -294.50115966796875,
      "logps/rejected": -425.34393310546875,
      "loss": 0.2232,
      "rewards/chosen": 1.6126306056976318,
      "rewards/margins": 9.582573890686035,
      "rewards/rejected": -7.969943046569824,
      "step": 600
    },
    {
      "epoch": 0.39,
      "grad_norm": 29.551122665405273,
      "kl": 0.0,
      "learning_rate": 3.406827880512091e-07,
      "logps/chosen": -267.5435485839844,
      "logps/rejected": -402.02362060546875,
      "loss": 0.2326,
      "rewards/chosen": 1.3285887241363525,
      "rewards/margins": 8.892502784729004,
      "rewards/rejected": -7.563913822174072,
      "step": 605
    },
    {
      "epoch": 0.39,
      "grad_norm": 31.5975284576416,
      "kl": 0.0,
      "learning_rate": 3.3890469416785205e-07,
      "logps/chosen": -254.05795288085938,
      "logps/rejected": -413.3831481933594,
      "loss": 0.2466,
      "rewards/chosen": 1.2830978631973267,
      "rewards/margins": 9.083808898925781,
      "rewards/rejected": -7.800711154937744,
      "step": 610
    },
    {
      "epoch": 0.39,
      "grad_norm": 32.5655632019043,
      "kl": 0.0,
      "learning_rate": 3.37126600284495e-07,
      "logps/chosen": -262.8067626953125,
      "logps/rejected": -406.58074951171875,
      "loss": 0.2192,
      "rewards/chosen": 1.6863523721694946,
      "rewards/margins": 9.860769271850586,
      "rewards/rejected": -8.174417495727539,
      "step": 615
    },
    {
      "epoch": 0.4,
      "grad_norm": 35.858154296875,
      "kl": 0.0,
      "learning_rate": 3.35348506401138e-07,
      "logps/chosen": -282.96319580078125,
      "logps/rejected": -426.3731384277344,
      "loss": 0.2181,
      "rewards/chosen": 1.4725250005722046,
      "rewards/margins": 9.46699333190918,
      "rewards/rejected": -7.994467735290527,
      "step": 620
    },
    {
      "epoch": 0.4,
      "grad_norm": 40.72344970703125,
      "kl": 0.0,
      "learning_rate": 3.335704125177809e-07,
      "logps/chosen": -268.4210510253906,
      "logps/rejected": -439.38671875,
      "loss": 0.2173,
      "rewards/chosen": 1.4876525402069092,
      "rewards/margins": 10.092061996459961,
      "rewards/rejected": -8.604409217834473,
      "step": 625
    },
    {
      "epoch": 0.4,
      "grad_norm": 30.59383773803711,
      "kl": 0.0,
      "learning_rate": 3.3179231863442384e-07,
      "logps/chosen": -287.0130310058594,
      "logps/rejected": -452.73974609375,
      "loss": 0.2039,
      "rewards/chosen": 1.455565333366394,
      "rewards/margins": 10.204021453857422,
      "rewards/rejected": -8.748456001281738,
      "step": 630
    },
    {
      "epoch": 0.41,
      "grad_norm": 33.28047561645508,
      "kl": 0.0,
      "learning_rate": 3.300142247510668e-07,
      "logps/chosen": -244.6796875,
      "logps/rejected": -446.35272216796875,
      "loss": 0.233,
      "rewards/chosen": 1.5680058002471924,
      "rewards/margins": 9.671673774719238,
      "rewards/rejected": -8.103668212890625,
      "step": 635
    },
    {
      "epoch": 0.41,
      "grad_norm": 30.61500358581543,
      "kl": 0.0,
      "learning_rate": 3.282361308677098e-07,
      "logps/chosen": -253.5943145751953,
      "logps/rejected": -432.730224609375,
      "loss": 0.2031,
      "rewards/chosen": 1.5605218410491943,
      "rewards/margins": 9.725998878479004,
      "rewards/rejected": -8.16547679901123,
      "step": 640
    },
    {
      "epoch": 0.41,
      "grad_norm": 33.63188934326172,
      "kl": 0.0,
      "learning_rate": 3.2645803698435276e-07,
      "logps/chosen": -271.66546630859375,
      "logps/rejected": -433.5645446777344,
      "loss": 0.2207,
      "rewards/chosen": 1.5964945554733276,
      "rewards/margins": 9.680837631225586,
      "rewards/rejected": -8.084342956542969,
      "step": 645
    },
    {
      "epoch": 0.42,
      "grad_norm": 33.40596389770508,
      "kl": 0.0,
      "learning_rate": 3.2467994310099573e-07,
      "logps/chosen": -285.18707275390625,
      "logps/rejected": -403.45782470703125,
      "loss": 0.2294,
      "rewards/chosen": 1.3258507251739502,
      "rewards/margins": 8.883944511413574,
      "rewards/rejected": -7.558093070983887,
      "step": 650
    },
    {
      "epoch": 0.42,
      "grad_norm": 30.218576431274414,
      "kl": 0.0,
      "learning_rate": 3.229018492176387e-07,
      "logps/chosen": -286.4705505371094,
      "logps/rejected": -406.6639099121094,
      "loss": 0.2219,
      "rewards/chosen": 1.627684235572815,
      "rewards/margins": 8.956671714782715,
      "rewards/rejected": -7.328988075256348,
      "step": 655
    },
    {
      "epoch": 0.42,
      "grad_norm": 32.87017059326172,
      "kl": 0.0,
      "learning_rate": 3.211237553342817e-07,
      "logps/chosen": -279.32794189453125,
      "logps/rejected": -414.5611267089844,
      "loss": 0.2147,
      "rewards/chosen": 1.4831647872924805,
      "rewards/margins": 9.314391136169434,
      "rewards/rejected": -7.831225395202637,
      "step": 660
    },
    {
      "epoch": 0.43,
      "grad_norm": 32.71424865722656,
      "kl": 0.0,
      "learning_rate": 3.193456614509246e-07,
      "logps/chosen": -304.61181640625,
      "logps/rejected": -443.0315856933594,
      "loss": 0.1965,
      "rewards/chosen": 1.8419644832611084,
      "rewards/margins": 10.106998443603516,
      "rewards/rejected": -8.265034675598145,
      "step": 665
    },
    {
      "epoch": 0.43,
      "grad_norm": 33.63492965698242,
      "kl": 0.0,
      "learning_rate": 3.175675675675675e-07,
      "logps/chosen": -310.9371032714844,
      "logps/rejected": -430.70367431640625,
      "loss": 0.2095,
      "rewards/chosen": 1.5251778364181519,
      "rewards/margins": 9.424676895141602,
      "rewards/rejected": -7.899499416351318,
      "step": 670
    },
    {
      "epoch": 0.43,
      "grad_norm": 28.59478759765625,
      "kl": 0.0,
      "learning_rate": 3.157894736842105e-07,
      "logps/chosen": -282.77020263671875,
      "logps/rejected": -390.97052001953125,
      "loss": 0.2338,
      "rewards/chosen": 1.6484334468841553,
      "rewards/margins": 9.223466873168945,
      "rewards/rejected": -7.575033664703369,
      "step": 675
    },
    {
      "epoch": 0.44,
      "grad_norm": 30.481687545776367,
      "kl": 0.0,
      "learning_rate": 3.1401137980085347e-07,
      "logps/chosen": -268.91680908203125,
      "logps/rejected": -405.0919494628906,
      "loss": 0.2032,
      "rewards/chosen": 2.029101610183716,
      "rewards/margins": 10.308364868164062,
      "rewards/rejected": -8.279263496398926,
      "step": 680
    },
    {
      "epoch": 0.44,
      "grad_norm": 31.664047241210938,
      "kl": 0.0,
      "learning_rate": 3.1223328591749644e-07,
      "logps/chosen": -276.86871337890625,
      "logps/rejected": -431.42156982421875,
      "loss": 0.1914,
      "rewards/chosen": 2.007537603378296,
      "rewards/margins": 10.265495300292969,
      "rewards/rejected": -8.257957458496094,
      "step": 685
    },
    {
      "epoch": 0.44,
      "grad_norm": 34.09368896484375,
      "kl": 0.0,
      "learning_rate": 3.104551920341394e-07,
      "logps/chosen": -281.3551025390625,
      "logps/rejected": -436.32196044921875,
      "loss": 0.2118,
      "rewards/chosen": 1.5368009805679321,
      "rewards/margins": 9.339311599731445,
      "rewards/rejected": -7.8025102615356445,
      "step": 690
    },
    {
      "epoch": 0.44,
      "grad_norm": 29.23426055908203,
      "kl": 0.0,
      "learning_rate": 3.0867709815078234e-07,
      "logps/chosen": -260.82012939453125,
      "logps/rejected": -410.68548583984375,
      "loss": 0.2196,
      "rewards/chosen": 1.3135395050048828,
      "rewards/margins": 9.747769355773926,
      "rewards/rejected": -8.434229850769043,
      "step": 695
    },
    {
      "epoch": 0.45,
      "grad_norm": 38.7562370300293,
      "kl": 0.0,
      "learning_rate": 3.068990042674253e-07,
      "logps/chosen": -308.45013427734375,
      "logps/rejected": -457.8919982910156,
      "loss": 0.2002,
      "rewards/chosen": 1.5923526287078857,
      "rewards/margins": 9.828653335571289,
      "rewards/rejected": -8.236300468444824,
      "step": 700
    },
    {
      "epoch": 0.45,
      "grad_norm": 30.01862907409668,
      "kl": 0.0,
      "learning_rate": 3.051209103840683e-07,
      "logps/chosen": -247.7996368408203,
      "logps/rejected": -413.0341796875,
      "loss": 0.1966,
      "rewards/chosen": 1.7354379892349243,
      "rewards/margins": 10.121583938598633,
      "rewards/rejected": -8.38614559173584,
      "step": 705
    },
    {
      "epoch": 0.45,
      "grad_norm": 30.769304275512695,
      "kl": 0.0,
      "learning_rate": 3.033428165007112e-07,
      "logps/chosen": -307.517822265625,
      "logps/rejected": -431.41375732421875,
      "loss": 0.2173,
      "rewards/chosen": 1.3397324085235596,
      "rewards/margins": 9.70555305480957,
      "rewards/rejected": -8.36582088470459,
      "step": 710
    },
    {
      "epoch": 0.46,
      "grad_norm": 35.207061767578125,
      "kl": 0.0,
      "learning_rate": 3.015647226173542e-07,
      "logps/chosen": -310.77899169921875,
      "logps/rejected": -399.6103515625,
      "loss": 0.2141,
      "rewards/chosen": 1.5721590518951416,
      "rewards/margins": 9.228109359741211,
      "rewards/rejected": -7.655949592590332,
      "step": 715
    },
    {
      "epoch": 0.46,
      "grad_norm": 28.209081649780273,
      "kl": 0.0,
      "learning_rate": 2.9978662873399715e-07,
      "logps/chosen": -259.1542663574219,
      "logps/rejected": -434.557373046875,
      "loss": 0.2125,
      "rewards/chosen": 1.7099430561065674,
      "rewards/margins": 9.807859420776367,
      "rewards/rejected": -8.097915649414062,
      "step": 720
    },
    {
      "epoch": 0.46,
      "grad_norm": 29.12105941772461,
      "kl": 0.0,
      "learning_rate": 2.9800853485064007e-07,
      "logps/chosen": -256.1063232421875,
      "logps/rejected": -449.45684814453125,
      "loss": 0.2144,
      "rewards/chosen": 1.423112154006958,
      "rewards/margins": 10.24951457977295,
      "rewards/rejected": -8.82640266418457,
      "step": 725
    },
    {
      "epoch": 0.47,
      "grad_norm": 31.086488723754883,
      "kl": 0.0,
      "learning_rate": 2.9623044096728305e-07,
      "logps/chosen": -332.27093505859375,
      "logps/rejected": -441.17303466796875,
      "loss": 0.1925,
      "rewards/chosen": 1.8641254901885986,
      "rewards/margins": 10.427138328552246,
      "rewards/rejected": -8.563013076782227,
      "step": 730
    },
    {
      "epoch": 0.47,
      "grad_norm": 30.249380111694336,
      "kl": 0.0,
      "learning_rate": 2.94452347083926e-07,
      "logps/chosen": -253.9059600830078,
      "logps/rejected": -411.62481689453125,
      "loss": 0.2097,
      "rewards/chosen": 1.827418565750122,
      "rewards/margins": 9.92874813079834,
      "rewards/rejected": -8.10132884979248,
      "step": 735
    },
    {
      "epoch": 0.47,
      "grad_norm": 34.48613739013672,
      "kl": 0.0,
      "learning_rate": 2.92674253200569e-07,
      "logps/chosen": -287.5168151855469,
      "logps/rejected": -488.13800048828125,
      "loss": 0.1975,
      "rewards/chosen": 1.7286927700042725,
      "rewards/margins": 10.497554779052734,
      "rewards/rejected": -8.768861770629883,
      "step": 740
    },
    {
      "epoch": 0.48,
      "grad_norm": 26.248523712158203,
      "kl": 0.0,
      "learning_rate": 2.9089615931721197e-07,
      "logps/chosen": -228.9366912841797,
      "logps/rejected": -417.3013610839844,
      "loss": 0.2257,
      "rewards/chosen": 1.364012360572815,
      "rewards/margins": 9.509763717651367,
      "rewards/rejected": -8.145750999450684,
      "step": 745
    },
    {
      "epoch": 0.48,
      "grad_norm": 29.836442947387695,
      "kl": 0.0,
      "learning_rate": 2.8911806543385494e-07,
      "logps/chosen": -267.7503967285156,
      "logps/rejected": -438.37884521484375,
      "loss": 0.1908,
      "rewards/chosen": 1.686102271080017,
      "rewards/margins": 10.126192092895508,
      "rewards/rejected": -8.440089225769043,
      "step": 750
    },
    {
      "epoch": 0.48,
      "grad_norm": 28.22798728942871,
      "kl": 0.0,
      "learning_rate": 2.873399715504978e-07,
      "logps/chosen": -268.92193603515625,
      "logps/rejected": -439.41021728515625,
      "loss": 0.1942,
      "rewards/chosen": 1.7055965662002563,
      "rewards/margins": 9.828977584838867,
      "rewards/rejected": -8.123380661010742,
      "step": 755
    },
    {
      "epoch": 0.49,
      "grad_norm": 30.871654510498047,
      "kl": 0.0,
      "learning_rate": 2.855618776671408e-07,
      "logps/chosen": -277.0738220214844,
      "logps/rejected": -409.88104248046875,
      "loss": 0.2134,
      "rewards/chosen": 1.6514962911605835,
      "rewards/margins": 10.07313346862793,
      "rewards/rejected": -8.421636581420898,
      "step": 760
    },
    {
      "epoch": 0.49,
      "grad_norm": 31.168502807617188,
      "kl": 0.0,
      "learning_rate": 2.8378378378378376e-07,
      "logps/chosen": -269.1263732910156,
      "logps/rejected": -439.88031005859375,
      "loss": 0.1708,
      "rewards/chosen": 1.8253052234649658,
      "rewards/margins": 10.730072021484375,
      "rewards/rejected": -8.904766082763672,
      "step": 765
    },
    {
      "epoch": 0.49,
      "grad_norm": 29.153135299682617,
      "kl": 0.0,
      "learning_rate": 2.8200568990042673e-07,
      "logps/chosen": -253.00991821289062,
      "logps/rejected": -433.29962158203125,
      "loss": 0.2071,
      "rewards/chosen": 1.6923249959945679,
      "rewards/margins": 9.882917404174805,
      "rewards/rejected": -8.190591812133789,
      "step": 770
    },
    {
      "epoch": 0.5,
      "grad_norm": 30.694440841674805,
      "kl": 0.0,
      "learning_rate": 2.802275960170697e-07,
      "logps/chosen": -291.7933044433594,
      "logps/rejected": -395.35870361328125,
      "loss": 0.2081,
      "rewards/chosen": 1.7577203512191772,
      "rewards/margins": 9.164716720581055,
      "rewards/rejected": -7.406996250152588,
      "step": 775
    },
    {
      "epoch": 0.5,
      "grad_norm": 33.14602279663086,
      "kl": 0.0,
      "learning_rate": 2.784495021337127e-07,
      "logps/chosen": -314.156005859375,
      "logps/rejected": -397.37933349609375,
      "loss": 0.2057,
      "rewards/chosen": 1.6907199621200562,
      "rewards/margins": 9.452787399291992,
      "rewards/rejected": -7.7620673179626465,
      "step": 780
    },
    {
      "epoch": 0.5,
      "grad_norm": 38.135986328125,
      "kl": 0.0,
      "learning_rate": 2.766714082503556e-07,
      "logps/chosen": -283.73907470703125,
      "logps/rejected": -405.26409912109375,
      "loss": 0.2019,
      "rewards/chosen": 1.7740901708602905,
      "rewards/margins": 10.007485389709473,
      "rewards/rejected": -8.23339557647705,
      "step": 785
    },
    {
      "epoch": 0.51,
      "grad_norm": 33.268096923828125,
      "kl": 0.0,
      "learning_rate": 2.7489331436699857e-07,
      "logps/chosen": -285.5155944824219,
      "logps/rejected": -444.726806640625,
      "loss": 0.2084,
      "rewards/chosen": 1.6661376953125,
      "rewards/margins": 9.84503173828125,
      "rewards/rejected": -8.178893089294434,
      "step": 790
    },
    {
      "epoch": 0.51,
      "grad_norm": 37.14868927001953,
      "kl": 0.0,
      "learning_rate": 2.7311522048364154e-07,
      "logps/chosen": -313.14813232421875,
      "logps/rejected": -434.33349609375,
      "loss": 0.216,
      "rewards/chosen": 1.766885757446289,
      "rewards/margins": 10.550158500671387,
      "rewards/rejected": -8.783272743225098,
      "step": 795
    },
    {
      "epoch": 0.51,
      "grad_norm": 27.936447143554688,
      "kl": 0.0,
      "learning_rate": 2.7133712660028446e-07,
      "logps/chosen": -328.349853515625,
      "logps/rejected": -440.5518493652344,
      "loss": 0.1845,
      "rewards/chosen": 2.068641424179077,
      "rewards/margins": 10.594073295593262,
      "rewards/rejected": -8.525431632995605,
      "step": 800
    },
    {
      "epoch": 0.52,
      "grad_norm": 34.39814376831055,
      "kl": 0.0,
      "learning_rate": 2.6955903271692744e-07,
      "logps/chosen": -265.58770751953125,
      "logps/rejected": -437.67791748046875,
      "loss": 0.2087,
      "rewards/chosen": 1.520298719406128,
      "rewards/margins": 10.759889602661133,
      "rewards/rejected": -9.239590644836426,
      "step": 805
    },
    {
      "epoch": 0.52,
      "grad_norm": 34.08564758300781,
      "kl": 0.0,
      "learning_rate": 2.677809388335704e-07,
      "logps/chosen": -292.2442321777344,
      "logps/rejected": -440.2151794433594,
      "loss": 0.1862,
      "rewards/chosen": 1.5941486358642578,
      "rewards/margins": 10.367888450622559,
      "rewards/rejected": -8.7737398147583,
      "step": 810
    },
    {
      "epoch": 0.52,
      "grad_norm": 28.865568161010742,
      "kl": 0.0,
      "learning_rate": 2.6600284495021333e-07,
      "logps/chosen": -287.80438232421875,
      "logps/rejected": -450.5841369628906,
      "loss": 0.1758,
      "rewards/chosen": 2.1789932250976562,
      "rewards/margins": 10.92069149017334,
      "rewards/rejected": -8.741698265075684,
      "step": 815
    },
    {
      "epoch": 0.52,
      "grad_norm": 30.55199432373047,
      "kl": 0.0,
      "learning_rate": 2.642247510668563e-07,
      "logps/chosen": -272.82916259765625,
      "logps/rejected": -385.1436462402344,
      "loss": 0.1674,
      "rewards/chosen": 2.123709201812744,
      "rewards/margins": 10.312447547912598,
      "rewards/rejected": -8.188737869262695,
      "step": 820
    },
    {
      "epoch": 0.53,
      "grad_norm": 29.896678924560547,
      "kl": 0.0,
      "learning_rate": 2.624466571834993e-07,
      "logps/chosen": -270.98382568359375,
      "logps/rejected": -423.6415100097656,
      "loss": 0.1591,
      "rewards/chosen": 1.9183921813964844,
      "rewards/margins": 10.909276008605957,
      "rewards/rejected": -8.990884780883789,
      "step": 825
    },
    {
      "epoch": 0.53,
      "grad_norm": 36.479026794433594,
      "kl": 0.0,
      "learning_rate": 2.6066856330014225e-07,
      "logps/chosen": -287.6585693359375,
      "logps/rejected": -416.476806640625,
      "loss": 0.2022,
      "rewards/chosen": 1.8126230239868164,
      "rewards/margins": 10.275026321411133,
      "rewards/rejected": -8.462403297424316,
      "step": 830
    },
    {
      "epoch": 0.53,
      "grad_norm": 29.992250442504883,
      "kl": 0.0,
      "learning_rate": 2.5889046941678523e-07,
      "logps/chosen": -292.3607482910156,
      "logps/rejected": -415.16961669921875,
      "loss": 0.1922,
      "rewards/chosen": 1.6790504455566406,
      "rewards/margins": 10.113423347473145,
      "rewards/rejected": -8.434372901916504,
      "step": 835
    },
    {
      "epoch": 0.54,
      "grad_norm": 27.18990707397461,
      "kl": 0.0,
      "learning_rate": 2.5711237553342815e-07,
      "logps/chosen": -300.2317810058594,
      "logps/rejected": -464.2012634277344,
      "loss": 0.1886,
      "rewards/chosen": 1.892055869102478,
      "rewards/margins": 11.013105392456055,
      "rewards/rejected": -9.121049880981445,
      "step": 840
    },
    {
      "epoch": 0.54,
      "grad_norm": 31.40631866455078,
      "kl": 0.0,
      "learning_rate": 2.5533428165007107e-07,
      "logps/chosen": -250.4829559326172,
      "logps/rejected": -420.1806640625,
      "loss": 0.173,
      "rewards/chosen": 1.9857699871063232,
      "rewards/margins": 10.808717727661133,
      "rewards/rejected": -8.82294750213623,
      "step": 845
    },
    {
      "epoch": 0.54,
      "grad_norm": 33.97587203979492,
      "kl": 0.0,
      "learning_rate": 2.5355618776671404e-07,
      "logps/chosen": -272.48077392578125,
      "logps/rejected": -404.07952880859375,
      "loss": 0.2001,
      "rewards/chosen": 1.8833481073379517,
      "rewards/margins": 9.897645950317383,
      "rewards/rejected": -8.014297485351562,
      "step": 850
    },
    {
      "epoch": 0.55,
      "grad_norm": 25.89413833618164,
      "kl": 0.0,
      "learning_rate": 2.51778093883357e-07,
      "logps/chosen": -262.54461669921875,
      "logps/rejected": -457.70391845703125,
      "loss": 0.1959,
      "rewards/chosen": 1.7342389822006226,
      "rewards/margins": 11.080788612365723,
      "rewards/rejected": -9.346549987792969,
      "step": 855
    },
    {
      "epoch": 0.55,
      "grad_norm": 26.989194869995117,
      "kl": 0.0,
      "learning_rate": 2.5e-07,
      "logps/chosen": -281.6241455078125,
      "logps/rejected": -437.0460510253906,
      "loss": 0.1866,
      "rewards/chosen": 2.0029733180999756,
      "rewards/margins": 10.16287899017334,
      "rewards/rejected": -8.159906387329102,
      "step": 860
    },
    {
      "epoch": 0.55,
      "grad_norm": 25.836423873901367,
      "kl": 0.0,
      "learning_rate": 2.4822190611664296e-07,
      "logps/chosen": -244.0792999267578,
      "logps/rejected": -401.6324768066406,
      "loss": 0.1968,
      "rewards/chosen": 2.024254322052002,
      "rewards/margins": 9.46214771270752,
      "rewards/rejected": -7.437892913818359,
      "step": 865
    },
    {
      "epoch": 0.56,
      "grad_norm": 33.39433670043945,
      "kl": 0.0,
      "learning_rate": 2.4644381223328594e-07,
      "logps/chosen": -309.8813781738281,
      "logps/rejected": -404.27130126953125,
      "loss": 0.186,
      "rewards/chosen": 2.3213727474212646,
      "rewards/margins": 10.310930252075195,
      "rewards/rejected": -7.98955774307251,
      "step": 870
    },
    {
      "epoch": 0.56,
      "grad_norm": 30.139936447143555,
      "kl": 0.0,
      "learning_rate": 2.4466571834992886e-07,
      "logps/chosen": -258.7943420410156,
      "logps/rejected": -407.911865234375,
      "loss": 0.1945,
      "rewards/chosen": 1.719043493270874,
      "rewards/margins": 9.795989990234375,
      "rewards/rejected": -8.076947212219238,
      "step": 875
    },
    {
      "epoch": 0.56,
      "grad_norm": 28.36025047302246,
      "kl": 0.0,
      "learning_rate": 2.4288762446657183e-07,
      "logps/chosen": -289.2504577636719,
      "logps/rejected": -441.6619567871094,
      "loss": 0.2112,
      "rewards/chosen": 1.8992469310760498,
      "rewards/margins": 10.501276969909668,
      "rewards/rejected": -8.602029800415039,
      "step": 880
    },
    {
      "epoch": 0.57,
      "grad_norm": 32.14490509033203,
      "kl": 0.0,
      "learning_rate": 2.411095305832148e-07,
      "logps/chosen": -261.7134704589844,
      "logps/rejected": -387.33746337890625,
      "loss": 0.2077,
      "rewards/chosen": 1.7473371028900146,
      "rewards/margins": 10.049051284790039,
      "rewards/rejected": -8.301712989807129,
      "step": 885
    },
    {
      "epoch": 0.57,
      "grad_norm": 26.520404815673828,
      "kl": 0.0,
      "learning_rate": 2.393314366998578e-07,
      "logps/chosen": -289.41156005859375,
      "logps/rejected": -428.0794982910156,
      "loss": 0.1848,
      "rewards/chosen": 1.9077268838882446,
      "rewards/margins": 10.590463638305664,
      "rewards/rejected": -8.68273639678955,
      "step": 890
    },
    {
      "epoch": 0.57,
      "grad_norm": 33.48114776611328,
      "kl": 0.0,
      "learning_rate": 2.375533428165007e-07,
      "logps/chosen": -222.7843780517578,
      "logps/rejected": -410.1954040527344,
      "loss": 0.2028,
      "rewards/chosen": 1.7176685333251953,
      "rewards/margins": 10.505666732788086,
      "rewards/rejected": -8.787999153137207,
      "step": 895
    },
    {
      "epoch": 0.58,
      "grad_norm": 27.82444190979004,
      "kl": 0.0,
      "learning_rate": 2.3577524893314365e-07,
      "logps/chosen": -287.64862060546875,
      "logps/rejected": -468.0263671875,
      "loss": 0.1755,
      "rewards/chosen": 1.7359222173690796,
      "rewards/margins": 10.884846687316895,
      "rewards/rejected": -9.14892578125,
      "step": 900
    },
    {
      "epoch": 0.58,
      "grad_norm": 25.083499908447266,
      "kl": 0.0,
      "learning_rate": 2.3399715504978662e-07,
      "logps/chosen": -240.41531372070312,
      "logps/rejected": -419.208740234375,
      "loss": 0.1942,
      "rewards/chosen": 1.688421607017517,
      "rewards/margins": 10.170954704284668,
      "rewards/rejected": -8.482534408569336,
      "step": 905
    },
    {
      "epoch": 0.58,
      "grad_norm": 30.813711166381836,
      "kl": 0.0,
      "learning_rate": 2.322190611664296e-07,
      "logps/chosen": -289.48516845703125,
      "logps/rejected": -434.6170349121094,
      "loss": 0.2155,
      "rewards/chosen": 1.709018349647522,
      "rewards/margins": 10.145505905151367,
      "rewards/rejected": -8.436488151550293,
      "step": 910
    },
    {
      "epoch": 0.59,
      "grad_norm": 32.20737075805664,
      "kl": 0.0,
      "learning_rate": 2.304409672830725e-07,
      "logps/chosen": -250.640380859375,
      "logps/rejected": -414.97930908203125,
      "loss": 0.2066,
      "rewards/chosen": 1.4742424488067627,
      "rewards/margins": 9.822726249694824,
      "rewards/rejected": -8.348483085632324,
      "step": 915
    },
    {
      "epoch": 0.59,
      "grad_norm": 36.69318389892578,
      "kl": 0.0,
      "learning_rate": 2.2866287339971549e-07,
      "logps/chosen": -299.4349060058594,
      "logps/rejected": -427.6712341308594,
      "loss": 0.1928,
      "rewards/chosen": 1.8161312341690063,
      "rewards/margins": 10.13882827758789,
      "rewards/rejected": -8.322697639465332,
      "step": 920
    },
    {
      "epoch": 0.59,
      "grad_norm": 32.18443298339844,
      "kl": 0.0,
      "learning_rate": 2.2688477951635846e-07,
      "logps/chosen": -243.90664672851562,
      "logps/rejected": -439.81182861328125,
      "loss": 0.193,
      "rewards/chosen": 1.9404337406158447,
      "rewards/margins": 10.789703369140625,
      "rewards/rejected": -8.84926986694336,
      "step": 925
    },
    {
      "epoch": 0.6,
      "grad_norm": 27.605424880981445,
      "kl": 0.0,
      "learning_rate": 2.251066856330014e-07,
      "logps/chosen": -309.91455078125,
      "logps/rejected": -394.070068359375,
      "loss": 0.169,
      "rewards/chosen": 1.9091155529022217,
      "rewards/margins": 10.408263206481934,
      "rewards/rejected": -8.499147415161133,
      "step": 930
    },
    {
      "epoch": 0.6,
      "grad_norm": 29.7116756439209,
      "kl": 0.0,
      "learning_rate": 2.2332859174964438e-07,
      "logps/chosen": -249.909423828125,
      "logps/rejected": -431.39007568359375,
      "loss": 0.2046,
      "rewards/chosen": 1.6568987369537354,
      "rewards/margins": 10.274327278137207,
      "rewards/rejected": -8.617426872253418,
      "step": 935
    },
    {
      "epoch": 0.6,
      "grad_norm": 22.485855102539062,
      "kl": 0.0,
      "learning_rate": 2.2155049786628733e-07,
      "logps/chosen": -263.9428405761719,
      "logps/rejected": -435.47802734375,
      "loss": 0.1713,
      "rewards/chosen": 1.8609449863433838,
      "rewards/margins": 11.174790382385254,
      "rewards/rejected": -9.31384563446045,
      "step": 940
    },
    {
      "epoch": 0.6,
      "grad_norm": 29.809690475463867,
      "kl": 0.0,
      "learning_rate": 2.1977240398293027e-07,
      "logps/chosen": -252.637451171875,
      "logps/rejected": -403.4510803222656,
      "loss": 0.1744,
      "rewards/chosen": 1.9656826257705688,
      "rewards/margins": 10.494858741760254,
      "rewards/rejected": -8.529176712036133,
      "step": 945
    },
    {
      "epoch": 0.61,
      "grad_norm": 26.905038833618164,
      "kl": 0.0,
      "learning_rate": 2.1799431009957325e-07,
      "logps/chosen": -269.87066650390625,
      "logps/rejected": -427.05584716796875,
      "loss": 0.1693,
      "rewards/chosen": 2.0619406700134277,
      "rewards/margins": 11.020936012268066,
      "rewards/rejected": -8.95899486541748,
      "step": 950
    },
    {
      "epoch": 0.61,
      "grad_norm": 29.09079933166504,
      "kl": 0.0,
      "learning_rate": 2.1621621621621622e-07,
      "logps/chosen": -288.0301208496094,
      "logps/rejected": -480.09796142578125,
      "loss": 0.2055,
      "rewards/chosen": 1.6792428493499756,
      "rewards/margins": 10.952609062194824,
      "rewards/rejected": -9.273366928100586,
      "step": 955
    },
    {
      "epoch": 0.61,
      "grad_norm": 28.329050064086914,
      "kl": 0.0,
      "learning_rate": 2.1443812233285914e-07,
      "logps/chosen": -249.3640594482422,
      "logps/rejected": -464.89483642578125,
      "loss": 0.1886,
      "rewards/chosen": 2.1078505516052246,
      "rewards/margins": 11.554121017456055,
      "rewards/rejected": -9.446270942687988,
      "step": 960
    },
    {
      "epoch": 0.62,
      "grad_norm": 27.633623123168945,
      "kl": 0.0,
      "learning_rate": 2.1266002844950212e-07,
      "logps/chosen": -282.1102294921875,
      "logps/rejected": -424.1873474121094,
      "loss": 0.1832,
      "rewards/chosen": 1.9060941934585571,
      "rewards/margins": 10.455194473266602,
      "rewards/rejected": -8.549099922180176,
      "step": 965
    },
    {
      "epoch": 0.62,
      "grad_norm": 28.65962028503418,
      "kl": 0.0,
      "learning_rate": 2.108819345661451e-07,
      "logps/chosen": -232.1042938232422,
      "logps/rejected": -440.0047912597656,
      "loss": 0.1883,
      "rewards/chosen": 1.5378611087799072,
      "rewards/margins": 10.000123023986816,
      "rewards/rejected": -8.462263107299805,
      "step": 970
    },
    {
      "epoch": 0.62,
      "grad_norm": 31.88002586364746,
      "kl": 0.0,
      "learning_rate": 2.0910384068278806e-07,
      "logps/chosen": -277.8791198730469,
      "logps/rejected": -397.03375244140625,
      "loss": 0.1909,
      "rewards/chosen": 1.89794921875,
      "rewards/margins": 11.083429336547852,
      "rewards/rejected": -9.185480117797852,
      "step": 975
    },
    {
      "epoch": 0.63,
      "grad_norm": 25.281034469604492,
      "kl": 0.0,
      "learning_rate": 2.0732574679943098e-07,
      "logps/chosen": -237.05593872070312,
      "logps/rejected": -407.40594482421875,
      "loss": 0.1685,
      "rewards/chosen": 2.0257680416107178,
      "rewards/margins": 10.262957572937012,
      "rewards/rejected": -8.237189292907715,
      "step": 980
    },
    {
      "epoch": 0.63,
      "grad_norm": 29.927499771118164,
      "kl": 0.0,
      "learning_rate": 2.0554765291607396e-07,
      "logps/chosen": -271.0245056152344,
      "logps/rejected": -412.03094482421875,
      "loss": 0.1945,
      "rewards/chosen": 1.5757964849472046,
      "rewards/margins": 9.921743392944336,
      "rewards/rejected": -8.345946311950684,
      "step": 985
    },
    {
      "epoch": 0.63,
      "grad_norm": 31.359935760498047,
      "kl": 0.0,
      "learning_rate": 2.0376955903271693e-07,
      "logps/chosen": -267.9440002441406,
      "logps/rejected": -442.4165954589844,
      "loss": 0.1952,
      "rewards/chosen": 1.9685808420181274,
      "rewards/margins": 10.528787612915039,
      "rewards/rejected": -8.560208320617676,
      "step": 990
    },
    {
      "epoch": 0.64,
      "grad_norm": 28.901763916015625,
      "kl": 0.0,
      "learning_rate": 2.0199146514935988e-07,
      "logps/chosen": -268.46533203125,
      "logps/rejected": -420.45538330078125,
      "loss": 0.1856,
      "rewards/chosen": 1.9667234420776367,
      "rewards/margins": 10.6937255859375,
      "rewards/rejected": -8.727002143859863,
      "step": 995
    },
    {
      "epoch": 0.64,
      "grad_norm": 25.9229736328125,
      "kl": 0.0,
      "learning_rate": 2.0021337126600283e-07,
      "logps/chosen": -287.6095886230469,
      "logps/rejected": -430.07647705078125,
      "loss": 0.1474,
      "rewards/chosen": 2.176244020462036,
      "rewards/margins": 11.091053009033203,
      "rewards/rejected": -8.91480827331543,
      "step": 1000
    },
    {
      "epoch": 0.64,
      "grad_norm": 23.820232391357422,
      "kl": 0.0,
      "learning_rate": 1.984352773826458e-07,
      "logps/chosen": -239.20166015625,
      "logps/rejected": -456.2108459472656,
      "loss": 0.1644,
      "rewards/chosen": 1.9704856872558594,
      "rewards/margins": 11.435120582580566,
      "rewards/rejected": -9.464634895324707,
      "step": 1005
    },
    {
      "epoch": 0.65,
      "grad_norm": 26.05778694152832,
      "kl": 0.0,
      "learning_rate": 1.9665718349928875e-07,
      "logps/chosen": -215.20626831054688,
      "logps/rejected": -431.2613220214844,
      "loss": 0.1632,
      "rewards/chosen": 1.852900505065918,
      "rewards/margins": 11.106436729431152,
      "rewards/rejected": -9.25353717803955,
      "step": 1010
    },
    {
      "epoch": 0.65,
      "grad_norm": 29.80303382873535,
      "kl": 0.0,
      "learning_rate": 1.9487908961593172e-07,
      "logps/chosen": -298.92431640625,
      "logps/rejected": -413.9326171875,
      "loss": 0.1737,
      "rewards/chosen": 2.1202621459960938,
      "rewards/margins": 11.231690406799316,
      "rewards/rejected": -9.111427307128906,
      "step": 1015
    },
    {
      "epoch": 0.65,
      "grad_norm": 27.96302032470703,
      "kl": 0.0,
      "learning_rate": 1.931009957325747e-07,
      "logps/chosen": -256.3444519042969,
      "logps/rejected": -434.3214416503906,
      "loss": 0.1876,
      "rewards/chosen": 1.7924187183380127,
      "rewards/margins": 10.647282600402832,
      "rewards/rejected": -8.854864120483398,
      "step": 1020
    },
    {
      "epoch": 0.66,
      "grad_norm": 20.548818588256836,
      "kl": 0.0,
      "learning_rate": 1.9132290184921761e-07,
      "logps/chosen": -277.95831298828125,
      "logps/rejected": -389.59893798828125,
      "loss": 0.1743,
      "rewards/chosen": 1.9753564596176147,
      "rewards/margins": 9.79386043548584,
      "rewards/rejected": -7.81850528717041,
      "step": 1025
    },
    {
      "epoch": 0.66,
      "grad_norm": 31.688438415527344,
      "kl": 0.0,
      "learning_rate": 1.895448079658606e-07,
      "logps/chosen": -237.60061645507812,
      "logps/rejected": -409.19232177734375,
      "loss": 0.1659,
      "rewards/chosen": 2.4839866161346436,
      "rewards/margins": 11.480685234069824,
      "rewards/rejected": -8.996698379516602,
      "step": 1030
    },
    {
      "epoch": 0.66,
      "grad_norm": 21.887004852294922,
      "kl": 0.0,
      "learning_rate": 1.8776671408250356e-07,
      "logps/chosen": -292.5876770019531,
      "logps/rejected": -428.93377685546875,
      "loss": 0.1665,
      "rewards/chosen": 2.078939199447632,
      "rewards/margins": 11.130914688110352,
      "rewards/rejected": -9.051974296569824,
      "step": 1035
    },
    {
      "epoch": 0.67,
      "grad_norm": 26.495267868041992,
      "kl": 0.0,
      "learning_rate": 1.859886201991465e-07,
      "logps/chosen": -267.95135498046875,
      "logps/rejected": -391.75408935546875,
      "loss": 0.1798,
      "rewards/chosen": 1.9513813257217407,
      "rewards/margins": 9.968805313110352,
      "rewards/rejected": -8.017423629760742,
      "step": 1040
    },
    {
      "epoch": 0.67,
      "grad_norm": 29.68768310546875,
      "kl": 0.0,
      "learning_rate": 1.8421052631578946e-07,
      "logps/chosen": -286.04925537109375,
      "logps/rejected": -390.62457275390625,
      "loss": 0.1773,
      "rewards/chosen": 2.327118158340454,
      "rewards/margins": 10.014382362365723,
      "rewards/rejected": -7.687264442443848,
      "step": 1045
    },
    {
      "epoch": 0.67,
      "grad_norm": 22.868907928466797,
      "kl": 0.0,
      "learning_rate": 1.8243243243243243e-07,
      "logps/chosen": -257.67413330078125,
      "logps/rejected": -435.93560791015625,
      "loss": 0.174,
      "rewards/chosen": 1.741437315940857,
      "rewards/margins": 10.20213508605957,
      "rewards/rejected": -8.46069622039795,
      "step": 1050
    },
    {
      "epoch": 0.67,
      "grad_norm": 26.902700424194336,
      "kl": 0.0,
      "learning_rate": 1.8065433854907538e-07,
      "logps/chosen": -243.84512329101562,
      "logps/rejected": -452.46405029296875,
      "loss": 0.1457,
      "rewards/chosen": 1.995959997177124,
      "rewards/margins": 11.115839004516602,
      "rewards/rejected": -9.119877815246582,
      "step": 1055
    },
    {
      "epoch": 0.68,
      "grad_norm": 30.123769760131836,
      "kl": 0.0,
      "learning_rate": 1.7887624466571835e-07,
      "logps/chosen": -273.56011962890625,
      "logps/rejected": -375.4561462402344,
      "loss": 0.1599,
      "rewards/chosen": 2.687070608139038,
      "rewards/margins": 10.254888534545898,
      "rewards/rejected": -7.567817687988281,
      "step": 1060
    },
    {
      "epoch": 0.68,
      "grad_norm": 33.86235809326172,
      "kl": 0.0,
      "learning_rate": 1.770981507823613e-07,
      "logps/chosen": -295.79144287109375,
      "logps/rejected": -406.8284606933594,
      "loss": 0.1904,
      "rewards/chosen": 1.9145209789276123,
      "rewards/margins": 10.37600326538086,
      "rewards/rejected": -8.461481094360352,
      "step": 1065
    },
    {
      "epoch": 0.68,
      "grad_norm": 34.696876525878906,
      "kl": 0.0,
      "learning_rate": 1.7532005689900424e-07,
      "logps/chosen": -278.31610107421875,
      "logps/rejected": -454.6402893066406,
      "loss": 0.1903,
      "rewards/chosen": 1.857825517654419,
      "rewards/margins": 10.412918090820312,
      "rewards/rejected": -8.555091857910156,
      "step": 1070
    },
    {
      "epoch": 0.69,
      "grad_norm": 28.04399871826172,
      "kl": 0.0,
      "learning_rate": 1.7354196301564722e-07,
      "logps/chosen": -279.0931091308594,
      "logps/rejected": -444.87322998046875,
      "loss": 0.174,
      "rewards/chosen": 1.844491720199585,
      "rewards/margins": 10.60900592803955,
      "rewards/rejected": -8.764513969421387,
      "step": 1075
    },
    {
      "epoch": 0.69,
      "grad_norm": 28.754440307617188,
      "kl": 0.0,
      "learning_rate": 1.717638691322902e-07,
      "logps/chosen": -277.6640625,
      "logps/rejected": -444.380615234375,
      "loss": 0.1684,
      "rewards/chosen": 1.9001327753067017,
      "rewards/margins": 10.396875381469727,
      "rewards/rejected": -8.496744155883789,
      "step": 1080
    },
    {
      "epoch": 0.69,
      "grad_norm": 23.833982467651367,
      "kl": 0.0,
      "learning_rate": 1.6998577524893314e-07,
      "logps/chosen": -271.98101806640625,
      "logps/rejected": -418.50384521484375,
      "loss": 0.176,
      "rewards/chosen": 1.8560775518417358,
      "rewards/margins": 9.872191429138184,
      "rewards/rejected": -8.016115188598633,
      "step": 1085
    },
    {
      "epoch": 0.7,
      "grad_norm": 24.5834903717041,
      "kl": 0.0,
      "learning_rate": 1.6820768136557609e-07,
      "logps/chosen": -253.29019165039062,
      "logps/rejected": -451.9681091308594,
      "loss": 0.1764,
      "rewards/chosen": 2.107499361038208,
      "rewards/margins": 10.960962295532227,
      "rewards/rejected": -8.853462219238281,
      "step": 1090
    },
    {
      "epoch": 0.7,
      "grad_norm": 25.093826293945312,
      "kl": 0.0,
      "learning_rate": 1.6642958748221906e-07,
      "logps/chosen": -296.2508239746094,
      "logps/rejected": -452.71624755859375,
      "loss": 0.1589,
      "rewards/chosen": 2.3644440174102783,
      "rewards/margins": 11.254218101501465,
      "rewards/rejected": -8.889775276184082,
      "step": 1095
    },
    {
      "epoch": 0.7,
      "grad_norm": 30.393264770507812,
      "kl": 0.0,
      "learning_rate": 1.64651493598862e-07,
      "logps/chosen": -244.84036254882812,
      "logps/rejected": -413.4102478027344,
      "loss": 0.1689,
      "rewards/chosen": 2.0877583026885986,
      "rewards/margins": 10.486618041992188,
      "rewards/rejected": -8.398859977722168,
      "step": 1100
    },
    {
      "epoch": 0.71,
      "grad_norm": 26.028972625732422,
      "kl": 0.0,
      "learning_rate": 1.6287339971550498e-07,
      "logps/chosen": -308.08587646484375,
      "logps/rejected": -427.04632568359375,
      "loss": 0.1642,
      "rewards/chosen": 2.250349760055542,
      "rewards/margins": 11.379538536071777,
      "rewards/rejected": -9.129188537597656,
      "step": 1105
    },
    {
      "epoch": 0.71,
      "grad_norm": 26.561185836791992,
      "kl": 0.0,
      "learning_rate": 1.6109530583214793e-07,
      "logps/chosen": -264.28485107421875,
      "logps/rejected": -427.21533203125,
      "loss": 0.1738,
      "rewards/chosen": 2.1204895973205566,
      "rewards/margins": 11.067842483520508,
      "rewards/rejected": -8.94735336303711,
      "step": 1110
    },
    {
      "epoch": 0.71,
      "grad_norm": 27.91498374938965,
      "kl": 0.0,
      "learning_rate": 1.5931721194879087e-07,
      "logps/chosen": -267.13275146484375,
      "logps/rejected": -440.71209716796875,
      "loss": 0.1777,
      "rewards/chosen": 1.9779777526855469,
      "rewards/margins": 10.832992553710938,
      "rewards/rejected": -8.85501480102539,
      "step": 1115
    },
    {
      "epoch": 0.72,
      "grad_norm": 22.371728897094727,
      "kl": 0.0,
      "learning_rate": 1.5753911806543385e-07,
      "logps/chosen": -253.9912109375,
      "logps/rejected": -447.35308837890625,
      "loss": 0.1532,
      "rewards/chosen": 2.4193108081817627,
      "rewards/margins": 11.426763534545898,
      "rewards/rejected": -9.007452964782715,
      "step": 1120
    },
    {
      "epoch": 0.72,
      "grad_norm": 32.35457229614258,
      "kl": 0.0,
      "learning_rate": 1.5576102418207682e-07,
      "logps/chosen": -263.775634765625,
      "logps/rejected": -407.1927795410156,
      "loss": 0.1735,
      "rewards/chosen": 2.165008068084717,
      "rewards/margins": 10.231494903564453,
      "rewards/rejected": -8.066487312316895,
      "step": 1125
    },
    {
      "epoch": 0.72,
      "grad_norm": 30.724489212036133,
      "kl": 0.0,
      "learning_rate": 1.5398293029871974e-07,
      "logps/chosen": -289.01605224609375,
      "logps/rejected": -425.20428466796875,
      "loss": 0.1649,
      "rewards/chosen": 2.4617397785186768,
      "rewards/margins": 11.305328369140625,
      "rewards/rejected": -8.843587875366211,
      "step": 1130
    },
    {
      "epoch": 0.73,
      "grad_norm": 23.74848747253418,
      "kl": 0.0,
      "learning_rate": 1.5220483641536272e-07,
      "logps/chosen": -260.9978942871094,
      "logps/rejected": -437.7108459472656,
      "loss": 0.1759,
      "rewards/chosen": 1.955735445022583,
      "rewards/margins": 10.639429092407227,
      "rewards/rejected": -8.683694839477539,
      "step": 1135
    },
    {
      "epoch": 0.73,
      "grad_norm": 30.680461883544922,
      "kl": 0.0,
      "learning_rate": 1.504267425320057e-07,
      "logps/chosen": -262.09991455078125,
      "logps/rejected": -451.2193908691406,
      "loss": 0.1629,
      "rewards/chosen": 1.9774525165557861,
      "rewards/margins": 11.371380805969238,
      "rewards/rejected": -9.393927574157715,
      "step": 1140
    },
    {
      "epoch": 0.73,
      "grad_norm": 28.192245483398438,
      "kl": 0.0,
      "learning_rate": 1.4864864864864866e-07,
      "logps/chosen": -264.33843994140625,
      "logps/rejected": -408.20050048828125,
      "loss": 0.2076,
      "rewards/chosen": 1.9607311487197876,
      "rewards/margins": 9.993951797485352,
      "rewards/rejected": -8.033220291137695,
      "step": 1145
    },
    {
      "epoch": 0.74,
      "grad_norm": 26.542776107788086,
      "kl": 0.0,
      "learning_rate": 1.4687055476529158e-07,
      "logps/chosen": -248.61154174804688,
      "logps/rejected": -396.5494079589844,
      "loss": 0.1501,
      "rewards/chosen": 2.4335556030273438,
      "rewards/margins": 11.315145492553711,
      "rewards/rejected": -8.88158893585205,
      "step": 1150
    },
    {
      "epoch": 0.74,
      "grad_norm": 26.956071853637695,
      "kl": 0.0,
      "learning_rate": 1.4509246088193456e-07,
      "logps/chosen": -240.13662719726562,
      "logps/rejected": -425.4685974121094,
      "loss": 0.17,
      "rewards/chosen": 2.171187162399292,
      "rewards/margins": 11.282613754272461,
      "rewards/rejected": -9.111427307128906,
      "step": 1155
    },
    {
      "epoch": 0.74,
      "grad_norm": 33.09626388549805,
      "kl": 0.0,
      "learning_rate": 1.4331436699857753e-07,
      "logps/chosen": -256.2728271484375,
      "logps/rejected": -413.5420837402344,
      "loss": 0.1812,
      "rewards/chosen": 1.7524036169052124,
      "rewards/margins": 10.1653470993042,
      "rewards/rejected": -8.412943840026855,
      "step": 1160
    },
    {
      "epoch": 0.75,
      "grad_norm": 22.70789909362793,
      "kl": 0.0,
      "learning_rate": 1.4153627311522048e-07,
      "logps/chosen": -270.51812744140625,
      "logps/rejected": -399.8717956542969,
      "loss": 0.1735,
      "rewards/chosen": 2.118551015853882,
      "rewards/margins": 10.402674674987793,
      "rewards/rejected": -8.284123420715332,
      "step": 1165
    },
    {
      "epoch": 0.75,
      "grad_norm": 21.700260162353516,
      "kl": 0.0,
      "learning_rate": 1.3975817923186345e-07,
      "logps/chosen": -270.01190185546875,
      "logps/rejected": -435.75439453125,
      "loss": 0.1542,
      "rewards/chosen": 2.2069079875946045,
      "rewards/margins": 11.315336227416992,
      "rewards/rejected": -9.108428955078125,
      "step": 1170
    },
    {
      "epoch": 0.75,
      "grad_norm": 29.502779006958008,
      "kl": 0.0,
      "learning_rate": 1.379800853485064e-07,
      "logps/chosen": -308.8605041503906,
      "logps/rejected": -433.614013671875,
      "loss": 0.167,
      "rewards/chosen": 1.9083741903305054,
      "rewards/margins": 11.036389350891113,
      "rewards/rejected": -9.12801456451416,
      "step": 1175
    },
    {
      "epoch": 0.75,
      "grad_norm": 25.997594833374023,
      "kl": 0.0,
      "learning_rate": 1.3620199146514935e-07,
      "logps/chosen": -294.05596923828125,
      "logps/rejected": -458.2794494628906,
      "loss": 0.1613,
      "rewards/chosen": 2.1892623901367188,
      "rewards/margins": 12.30145263671875,
      "rewards/rejected": -10.112190246582031,
      "step": 1180
    },
    {
      "epoch": 0.76,
      "grad_norm": 28.619224548339844,
      "kl": 0.0,
      "learning_rate": 1.3442389758179232e-07,
      "logps/chosen": -271.43994140625,
      "logps/rejected": -424.03033447265625,
      "loss": 0.19,
      "rewards/chosen": 1.8617703914642334,
      "rewards/margins": 10.569725036621094,
      "rewards/rejected": -8.707954406738281,
      "step": 1185
    },
    {
      "epoch": 0.76,
      "grad_norm": 35.789634704589844,
      "kl": 0.0,
      "learning_rate": 1.326458036984353e-07,
      "logps/chosen": -239.49771118164062,
      "logps/rejected": -422.3915100097656,
      "loss": 0.2053,
      "rewards/chosen": 1.7723499536514282,
      "rewards/margins": 10.303890228271484,
      "rewards/rejected": -8.531539916992188,
      "step": 1190
    },
    {
      "epoch": 0.76,
      "grad_norm": 28.460819244384766,
      "kl": 0.0,
      "learning_rate": 1.3086770981507821e-07,
      "logps/chosen": -275.1038818359375,
      "logps/rejected": -407.48260498046875,
      "loss": 0.1854,
      "rewards/chosen": 2.025543689727783,
      "rewards/margins": 10.616659164428711,
      "rewards/rejected": -8.591115951538086,
      "step": 1195
    },
    {
      "epoch": 0.77,
      "grad_norm": 32.08914566040039,
      "kl": 0.0,
      "learning_rate": 1.290896159317212e-07,
      "logps/chosen": -220.82662963867188,
      "logps/rejected": -450.47454833984375,
      "loss": 0.1691,
      "rewards/chosen": 1.8781731128692627,
      "rewards/margins": 11.196914672851562,
      "rewards/rejected": -9.318740844726562,
      "step": 1200
    },
    {
      "epoch": 0.77,
      "grad_norm": 24.067890167236328,
      "kl": 0.0,
      "learning_rate": 1.2731152204836416e-07,
      "logps/chosen": -248.81246948242188,
      "logps/rejected": -438.7205505371094,
      "loss": 0.1745,
      "rewards/chosen": 2.0623905658721924,
      "rewards/margins": 11.863210678100586,
      "rewards/rejected": -9.800821304321289,
      "step": 1205
    },
    {
      "epoch": 0.77,
      "grad_norm": 19.57704734802246,
      "kl": 0.0,
      "learning_rate": 1.255334281650071e-07,
      "logps/chosen": -251.44198608398438,
      "logps/rejected": -444.56219482421875,
      "loss": 0.1694,
      "rewards/chosen": 2.024569272994995,
      "rewards/margins": 11.268827438354492,
      "rewards/rejected": -9.244257926940918,
      "step": 1210
    },
    {
      "epoch": 0.78,
      "grad_norm": 26.092870712280273,
      "kl": 0.0,
      "learning_rate": 1.2375533428165005e-07,
      "logps/chosen": -252.7013702392578,
      "logps/rejected": -421.412841796875,
      "loss": 0.1779,
      "rewards/chosen": 2.1870219707489014,
      "rewards/margins": 10.504962921142578,
      "rewards/rejected": -8.317939758300781,
      "step": 1215
    },
    {
      "epoch": 0.78,
      "grad_norm": 27.678754806518555,
      "kl": 0.0,
      "learning_rate": 1.2197724039829303e-07,
      "logps/chosen": -272.28997802734375,
      "logps/rejected": -456.04644775390625,
      "loss": 0.1578,
      "rewards/chosen": 2.1497280597686768,
      "rewards/margins": 11.176363945007324,
      "rewards/rejected": -9.026636123657227,
      "step": 1220
    },
    {
      "epoch": 0.78,
      "grad_norm": 29.93063735961914,
      "kl": 0.0,
      "learning_rate": 1.2019914651493598e-07,
      "logps/chosen": -297.31317138671875,
      "logps/rejected": -434.39874267578125,
      "loss": 0.1417,
      "rewards/chosen": 2.197824001312256,
      "rewards/margins": 11.505438804626465,
      "rewards/rejected": -9.30761432647705,
      "step": 1225
    },
    {
      "epoch": 0.79,
      "grad_norm": 31.17092514038086,
      "kl": 0.0,
      "learning_rate": 1.1842105263157894e-07,
      "logps/chosen": -284.43408203125,
      "logps/rejected": -430.77850341796875,
      "loss": 0.1479,
      "rewards/chosen": 2.323302745819092,
      "rewards/margins": 11.516798973083496,
      "rewards/rejected": -9.193496704101562,
      "step": 1230
    },
    {
      "epoch": 0.79,
      "grad_norm": 21.476613998413086,
      "kl": 0.0,
      "learning_rate": 1.166429587482219e-07,
      "logps/chosen": -237.2047576904297,
      "logps/rejected": -418.5458984375,
      "loss": 0.1591,
      "rewards/chosen": 2.236788034439087,
      "rewards/margins": 10.972362518310547,
      "rewards/rejected": -8.735574722290039,
      "step": 1235
    },
    {
      "epoch": 0.79,
      "grad_norm": 27.719011306762695,
      "kl": 0.0,
      "learning_rate": 1.1486486486486487e-07,
      "logps/chosen": -267.37603759765625,
      "logps/rejected": -426.73822021484375,
      "loss": 0.1792,
      "rewards/chosen": 2.1171622276306152,
      "rewards/margins": 11.552632331848145,
      "rewards/rejected": -9.435468673706055,
      "step": 1240
    },
    {
      "epoch": 0.8,
      "grad_norm": 22.965457916259766,
      "kl": 0.0,
      "learning_rate": 1.1308677098150782e-07,
      "logps/chosen": -294.0093688964844,
      "logps/rejected": -444.3059997558594,
      "loss": 0.1481,
      "rewards/chosen": 2.485621929168701,
      "rewards/margins": 11.076322555541992,
      "rewards/rejected": -8.59070110321045,
      "step": 1245
    },
    {
      "epoch": 0.8,
      "grad_norm": 33.64476013183594,
      "kl": 0.0,
      "learning_rate": 1.1130867709815078e-07,
      "logps/chosen": -279.5322265625,
      "logps/rejected": -423.5594787597656,
      "loss": 0.1573,
      "rewards/chosen": 2.5543296337127686,
      "rewards/margins": 11.313433647155762,
      "rewards/rejected": -8.759103775024414,
      "step": 1250
    },
    {
      "epoch": 0.8,
      "grad_norm": 31.360397338867188,
      "kl": 0.0,
      "learning_rate": 1.0953058321479374e-07,
      "logps/chosen": -257.98211669921875,
      "logps/rejected": -386.41375732421875,
      "loss": 0.1399,
      "rewards/chosen": 2.3777503967285156,
      "rewards/margins": 10.759562492370605,
      "rewards/rejected": -8.38181209564209,
      "step": 1255
    },
    {
      "epoch": 0.81,
      "grad_norm": 23.901107788085938,
      "kl": 0.0,
      "learning_rate": 1.077524893314367e-07,
      "logps/chosen": -222.93588256835938,
      "logps/rejected": -408.0344543457031,
      "loss": 0.1802,
      "rewards/chosen": 1.903738021850586,
      "rewards/margins": 10.58789348602295,
      "rewards/rejected": -8.684155464172363,
      "step": 1260
    },
    {
      "epoch": 0.81,
      "grad_norm": 30.361534118652344,
      "kl": 0.0,
      "learning_rate": 1.0597439544807964e-07,
      "logps/chosen": -246.6664276123047,
      "logps/rejected": -412.19891357421875,
      "loss": 0.1639,
      "rewards/chosen": 2.1586594581604004,
      "rewards/margins": 11.214345932006836,
      "rewards/rejected": -9.055686950683594,
      "step": 1265
    },
    {
      "epoch": 0.81,
      "grad_norm": 28.034950256347656,
      "kl": 0.0,
      "learning_rate": 1.0419630156472262e-07,
      "logps/chosen": -343.211669921875,
      "logps/rejected": -472.8130798339844,
      "loss": 0.1302,
      "rewards/chosen": 2.3979649543762207,
      "rewards/margins": 12.091375350952148,
      "rewards/rejected": -9.693410873413086,
      "step": 1270
    },
    {
      "epoch": 0.82,
      "grad_norm": 32.206668853759766,
      "kl": 0.0,
      "learning_rate": 1.0241820768136557e-07,
      "logps/chosen": -293.3280334472656,
      "logps/rejected": -412.0704040527344,
      "loss": 0.1618,
      "rewards/chosen": 2.4007017612457275,
      "rewards/margins": 11.050657272338867,
      "rewards/rejected": -8.649954795837402,
      "step": 1275
    },
    {
      "epoch": 0.82,
      "grad_norm": 26.69512367248535,
      "kl": 0.0,
      "learning_rate": 1.0064011379800854e-07,
      "logps/chosen": -264.37652587890625,
      "logps/rejected": -443.7510681152344,
      "loss": 0.144,
      "rewards/chosen": 2.442653179168701,
      "rewards/margins": 11.348746299743652,
      "rewards/rejected": -8.90609359741211,
      "step": 1280
    },
    {
      "epoch": 0.82,
      "grad_norm": 17.890409469604492,
      "kl": 0.0,
      "learning_rate": 9.886201991465149e-08,
      "logps/chosen": -276.50457763671875,
      "logps/rejected": -464.80499267578125,
      "loss": 0.1395,
      "rewards/chosen": 2.2761783599853516,
      "rewards/margins": 11.834190368652344,
      "rewards/rejected": -9.558012008666992,
      "step": 1285
    },
    {
      "epoch": 0.83,
      "grad_norm": 25.79632568359375,
      "kl": 0.0,
      "learning_rate": 9.708392603129445e-08,
      "logps/chosen": -278.9715881347656,
      "logps/rejected": -442.2305603027344,
      "loss": 0.1614,
      "rewards/chosen": 2.176389217376709,
      "rewards/margins": 11.664901733398438,
      "rewards/rejected": -9.488512992858887,
      "step": 1290
    },
    {
      "epoch": 0.83,
      "grad_norm": 29.441097259521484,
      "kl": 0.0,
      "learning_rate": 9.530583214793741e-08,
      "logps/chosen": -276.7694396972656,
      "logps/rejected": -402.5328063964844,
      "loss": 0.155,
      "rewards/chosen": 2.361067533493042,
      "rewards/margins": 10.893728256225586,
      "rewards/rejected": -8.532661437988281,
      "step": 1295
    },
    {
      "epoch": 0.83,
      "grad_norm": 24.844789505004883,
      "kl": 0.0,
      "learning_rate": 9.352773826458037e-08,
      "logps/chosen": -289.9748229980469,
      "logps/rejected": -440.4497985839844,
      "loss": 0.1708,
      "rewards/chosen": 2.353919506072998,
      "rewards/margins": 11.433218002319336,
      "rewards/rejected": -9.079299926757812,
      "step": 1300
    },
    {
      "epoch": 0.83,
      "grad_norm": 28.899717330932617,
      "kl": 0.0,
      "learning_rate": 9.174964438122331e-08,
      "logps/chosen": -312.39605712890625,
      "logps/rejected": -427.78570556640625,
      "loss": 0.1639,
      "rewards/chosen": 2.488450527191162,
      "rewards/margins": 11.697278022766113,
      "rewards/rejected": -9.208827018737793,
      "step": 1305
    },
    {
      "epoch": 0.84,
      "grad_norm": 25.086254119873047,
      "kl": 0.0,
      "learning_rate": 8.997155049786629e-08,
      "logps/chosen": -273.1850280761719,
      "logps/rejected": -413.96282958984375,
      "loss": 0.1535,
      "rewards/chosen": 2.2840752601623535,
      "rewards/margins": 11.001981735229492,
      "rewards/rejected": -8.71790599822998,
      "step": 1310
    },
    {
      "epoch": 0.84,
      "grad_norm": 24.174297332763672,
      "kl": 0.0,
      "learning_rate": 8.819345661450925e-08,
      "logps/chosen": -260.79534912109375,
      "logps/rejected": -404.79302978515625,
      "loss": 0.1608,
      "rewards/chosen": 2.0041592121124268,
      "rewards/margins": 10.949081420898438,
      "rewards/rejected": -8.944923400878906,
      "step": 1315
    },
    {
      "epoch": 0.84,
      "grad_norm": 22.477380752563477,
      "kl": 0.0,
      "learning_rate": 8.64153627311522e-08,
      "logps/chosen": -287.4140625,
      "logps/rejected": -402.8482360839844,
      "loss": 0.1478,
      "rewards/chosen": 2.44392728805542,
      "rewards/margins": 11.139135360717773,
      "rewards/rejected": -8.695208549499512,
      "step": 1320
    },
    {
      "epoch": 0.85,
      "grad_norm": 24.919546127319336,
      "kl": 0.0,
      "learning_rate": 8.463726884779517e-08,
      "logps/chosen": -331.8410949707031,
      "logps/rejected": -441.4012756347656,
      "loss": 0.1521,
      "rewards/chosen": 2.6449999809265137,
      "rewards/margins": 11.747162818908691,
      "rewards/rejected": -9.102163314819336,
      "step": 1325
    },
    {
      "epoch": 0.85,
      "grad_norm": 24.160480499267578,
      "kl": 0.0,
      "learning_rate": 8.285917496443812e-08,
      "logps/chosen": -315.6636657714844,
      "logps/rejected": -459.8597717285156,
      "loss": 0.1475,
      "rewards/chosen": 2.497434139251709,
      "rewards/margins": 11.85330867767334,
      "rewards/rejected": -9.355875015258789,
      "step": 1330
    },
    {
      "epoch": 0.85,
      "grad_norm": 28.45322608947754,
      "kl": 0.0,
      "learning_rate": 8.108108108108108e-08,
      "logps/chosen": -248.135498046875,
      "logps/rejected": -418.92388916015625,
      "loss": 0.1706,
      "rewards/chosen": 2.0972299575805664,
      "rewards/margins": 10.873918533325195,
      "rewards/rejected": -8.776689529418945,
      "step": 1335
    },
    {
      "epoch": 0.86,
      "grad_norm": 24.973793029785156,
      "kl": 0.0,
      "learning_rate": 7.930298719772404e-08,
      "logps/chosen": -274.0903015136719,
      "logps/rejected": -436.2459411621094,
      "loss": 0.1563,
      "rewards/chosen": 2.0231974124908447,
      "rewards/margins": 10.445603370666504,
      "rewards/rejected": -8.422405242919922,
      "step": 1340
    },
    {
      "epoch": 0.86,
      "grad_norm": 26.009220123291016,
      "kl": 0.0,
      "learning_rate": 7.7524893314367e-08,
      "logps/chosen": -303.43560791015625,
      "logps/rejected": -419.0430603027344,
      "loss": 0.1562,
      "rewards/chosen": 2.5205211639404297,
      "rewards/margins": 11.333113670349121,
      "rewards/rejected": -8.812593460083008,
      "step": 1345
    },
    {
      "epoch": 0.86,
      "grad_norm": 31.752500534057617,
      "kl": 0.0,
      "learning_rate": 7.574679943100994e-08,
      "logps/chosen": -257.14483642578125,
      "logps/rejected": -424.982666015625,
      "loss": 0.1654,
      "rewards/chosen": 2.0342319011688232,
      "rewards/margins": 10.941479682922363,
      "rewards/rejected": -8.907247543334961,
      "step": 1350
    },
    {
      "epoch": 0.87,
      "grad_norm": 23.12604522705078,
      "kl": 0.0,
      "learning_rate": 7.396870554765292e-08,
      "logps/chosen": -269.96258544921875,
      "logps/rejected": -391.7086181640625,
      "loss": 0.1525,
      "rewards/chosen": 2.2532811164855957,
      "rewards/margins": 11.61485481262207,
      "rewards/rejected": -9.361574172973633,
      "step": 1355
    },
    {
      "epoch": 0.87,
      "grad_norm": 23.052330017089844,
      "kl": 0.0,
      "learning_rate": 7.219061166429587e-08,
      "logps/chosen": -262.48516845703125,
      "logps/rejected": -439.3185119628906,
      "loss": 0.1466,
      "rewards/chosen": 2.3807787895202637,
      "rewards/margins": 11.63065242767334,
      "rewards/rejected": -9.249873161315918,
      "step": 1360
    },
    {
      "epoch": 0.87,
      "grad_norm": 24.81175422668457,
      "kl": 0.0,
      "learning_rate": 7.041251778093883e-08,
      "logps/chosen": -269.81109619140625,
      "logps/rejected": -454.2550354003906,
      "loss": 0.1531,
      "rewards/chosen": 2.286533832550049,
      "rewards/margins": 12.61316967010498,
      "rewards/rejected": -10.326637268066406,
      "step": 1365
    },
    {
      "epoch": 0.88,
      "grad_norm": 28.95956802368164,
      "kl": 0.0,
      "learning_rate": 6.863442389758179e-08,
      "logps/chosen": -281.4117126464844,
      "logps/rejected": -444.41888427734375,
      "loss": 0.1483,
      "rewards/chosen": 2.1853625774383545,
      "rewards/margins": 11.702371597290039,
      "rewards/rejected": -9.517007827758789,
      "step": 1370
    },
    {
      "epoch": 0.88,
      "grad_norm": 21.27782440185547,
      "kl": 0.0,
      "learning_rate": 6.685633001422475e-08,
      "logps/chosen": -243.04190063476562,
      "logps/rejected": -399.4400634765625,
      "loss": 0.1326,
      "rewards/chosen": 2.5309996604919434,
      "rewards/margins": 11.317906379699707,
      "rewards/rejected": -8.786906242370605,
      "step": 1375
    },
    {
      "epoch": 0.88,
      "grad_norm": 27.220003128051758,
      "kl": 0.0,
      "learning_rate": 6.507823613086771e-08,
      "logps/chosen": -305.0284118652344,
      "logps/rejected": -460.77850341796875,
      "loss": 0.1451,
      "rewards/chosen": 2.280113935470581,
      "rewards/margins": 12.219154357910156,
      "rewards/rejected": -9.93903923034668,
      "step": 1380
    },
    {
      "epoch": 0.89,
      "grad_norm": 26.04877471923828,
      "kl": 0.0,
      "learning_rate": 6.330014224751067e-08,
      "logps/chosen": -299.484619140625,
      "logps/rejected": -444.88055419921875,
      "loss": 0.1383,
      "rewards/chosen": 2.363182544708252,
      "rewards/margins": 12.05009937286377,
      "rewards/rejected": -9.686917304992676,
      "step": 1385
    },
    {
      "epoch": 0.89,
      "grad_norm": 15.905016899108887,
      "kl": 0.0,
      "learning_rate": 6.152204836415363e-08,
      "logps/chosen": -253.1654815673828,
      "logps/rejected": -438.5511169433594,
      "loss": 0.1442,
      "rewards/chosen": 2.5684008598327637,
      "rewards/margins": 11.512083053588867,
      "rewards/rejected": -8.943681716918945,
      "step": 1390
    },
    {
      "epoch": 0.89,
      "grad_norm": 18.374181747436523,
      "kl": 0.0,
      "learning_rate": 5.974395448079659e-08,
      "logps/chosen": -240.8791046142578,
      "logps/rejected": -411.80487060546875,
      "loss": 0.1513,
      "rewards/chosen": 2.337771415710449,
      "rewards/margins": 11.283363342285156,
      "rewards/rejected": -8.945591926574707,
      "step": 1395
    },
    {
      "epoch": 0.9,
      "grad_norm": 30.287019729614258,
      "kl": 0.0,
      "learning_rate": 5.796586059743954e-08,
      "logps/chosen": -293.8108825683594,
      "logps/rejected": -451.5350646972656,
      "loss": 0.1654,
      "rewards/chosen": 1.9501450061798096,
      "rewards/margins": 11.551190376281738,
      "rewards/rejected": -9.601045608520508,
      "step": 1400
    },
    {
      "epoch": 0.9,
      "grad_norm": 24.658693313598633,
      "kl": 0.0,
      "learning_rate": 5.61877667140825e-08,
      "logps/chosen": -258.39886474609375,
      "logps/rejected": -444.2666015625,
      "loss": 0.1634,
      "rewards/chosen": 2.189380407333374,
      "rewards/margins": 11.383289337158203,
      "rewards/rejected": -9.19390869140625,
      "step": 1405
    },
    {
      "epoch": 0.9,
      "grad_norm": 24.983081817626953,
      "kl": 0.0,
      "learning_rate": 5.4409672830725456e-08,
      "logps/chosen": -292.2581787109375,
      "logps/rejected": -433.89453125,
      "loss": 0.1553,
      "rewards/chosen": 2.1917474269866943,
      "rewards/margins": 11.13335132598877,
      "rewards/rejected": -8.941603660583496,
      "step": 1410
    },
    {
      "epoch": 0.91,
      "grad_norm": 30.233684539794922,
      "kl": 0.0,
      "learning_rate": 5.2631578947368416e-08,
      "logps/chosen": -298.6884765625,
      "logps/rejected": -440.4461975097656,
      "loss": 0.1469,
      "rewards/chosen": 2.396724224090576,
      "rewards/margins": 12.125951766967773,
      "rewards/rejected": -9.729228019714355,
      "step": 1415
    },
    {
      "epoch": 0.91,
      "grad_norm": 27.67980194091797,
      "kl": 0.0,
      "learning_rate": 5.0853485064011376e-08,
      "logps/chosen": -293.10589599609375,
      "logps/rejected": -429.2474060058594,
      "loss": 0.1648,
      "rewards/chosen": 2.3396992683410645,
      "rewards/margins": 11.225849151611328,
      "rewards/rejected": -8.886149406433105,
      "step": 1420
    },
    {
      "epoch": 0.91,
      "grad_norm": 29.193660736083984,
      "kl": 0.0,
      "learning_rate": 4.9075391180654337e-08,
      "logps/chosen": -270.0514221191406,
      "logps/rejected": -460.9169921875,
      "loss": 0.177,
      "rewards/chosen": 1.9409351348876953,
      "rewards/margins": 11.288286209106445,
      "rewards/rejected": -9.34735107421875,
      "step": 1425
    },
    {
      "epoch": 0.91,
      "grad_norm": 25.77621078491211,
      "kl": 0.0,
      "learning_rate": 4.72972972972973e-08,
      "logps/chosen": -262.68060302734375,
      "logps/rejected": -414.3082580566406,
      "loss": 0.1517,
      "rewards/chosen": 2.055493116378784,
      "rewards/margins": 10.508028030395508,
      "rewards/rejected": -8.452535629272461,
      "step": 1430
    },
    {
      "epoch": 0.92,
      "grad_norm": 39.052642822265625,
      "kl": 0.0,
      "learning_rate": 4.551920341394026e-08,
      "logps/chosen": -323.07977294921875,
      "logps/rejected": -415.31268310546875,
      "loss": 0.1704,
      "rewards/chosen": 2.5183842182159424,
      "rewards/margins": 11.62377643585205,
      "rewards/rejected": -9.105392456054688,
      "step": 1435
    },
    {
      "epoch": 0.92,
      "grad_norm": 27.706132888793945,
      "kl": 0.0,
      "learning_rate": 4.374110953058322e-08,
      "logps/chosen": -279.017822265625,
      "logps/rejected": -474.2362365722656,
      "loss": 0.1525,
      "rewards/chosen": 2.0204992294311523,
      "rewards/margins": 11.512517929077148,
      "rewards/rejected": -9.49201774597168,
      "step": 1440
    },
    {
      "epoch": 0.92,
      "grad_norm": 24.838924407958984,
      "kl": 0.0,
      "learning_rate": 4.196301564722617e-08,
      "logps/chosen": -285.550048828125,
      "logps/rejected": -462.3155822753906,
      "loss": 0.1689,
      "rewards/chosen": 2.1153082847595215,
      "rewards/margins": 11.936882019042969,
      "rewards/rejected": -9.821573257446289,
      "step": 1445
    },
    {
      "epoch": 0.93,
      "grad_norm": 34.149696350097656,
      "kl": 0.0,
      "learning_rate": 4.018492176386913e-08,
      "logps/chosen": -261.6683349609375,
      "logps/rejected": -418.8206481933594,
      "loss": 0.1861,
      "rewards/chosen": 1.9666168689727783,
      "rewards/margins": 10.487935066223145,
      "rewards/rejected": -8.521318435668945,
      "step": 1450
    },
    {
      "epoch": 0.93,
      "grad_norm": 23.415525436401367,
      "kl": 0.0,
      "learning_rate": 3.840682788051209e-08,
      "logps/chosen": -239.5380096435547,
      "logps/rejected": -418.8392639160156,
      "loss": 0.1519,
      "rewards/chosen": 2.0457942485809326,
      "rewards/margins": 11.141304969787598,
      "rewards/rejected": -9.095510482788086,
      "step": 1455
    },
    {
      "epoch": 0.93,
      "grad_norm": 18.416677474975586,
      "kl": 0.0,
      "learning_rate": 3.6628733997155046e-08,
      "logps/chosen": -293.07208251953125,
      "logps/rejected": -434.031494140625,
      "loss": 0.1416,
      "rewards/chosen": 2.377678632736206,
      "rewards/margins": 11.937261581420898,
      "rewards/rejected": -9.55958366394043,
      "step": 1460
    },
    {
      "epoch": 0.94,
      "grad_norm": 24.344444274902344,
      "kl": 0.0,
      "learning_rate": 3.4850640113798006e-08,
      "logps/chosen": -297.32183837890625,
      "logps/rejected": -437.09100341796875,
      "loss": 0.1229,
      "rewards/chosen": 2.8577122688293457,
      "rewards/margins": 12.127481460571289,
      "rewards/rejected": -9.269769668579102,
      "step": 1465
    },
    {
      "epoch": 0.94,
      "grad_norm": 30.76423454284668,
      "kl": 0.0,
      "learning_rate": 3.3072546230440967e-08,
      "logps/chosen": -279.06219482421875,
      "logps/rejected": -445.42156982421875,
      "loss": 0.1722,
      "rewards/chosen": 2.1692028045654297,
      "rewards/margins": 11.08595085144043,
      "rewards/rejected": -8.916748046875,
      "step": 1470
    },
    {
      "epoch": 0.94,
      "grad_norm": 22.21062660217285,
      "kl": 0.0,
      "learning_rate": 3.129445234708392e-08,
      "logps/chosen": -255.56723022460938,
      "logps/rejected": -437.4034729003906,
      "loss": 0.1455,
      "rewards/chosen": 2.3138813972473145,
      "rewards/margins": 11.781050682067871,
      "rewards/rejected": -9.467168807983398,
      "step": 1475
    },
    {
      "epoch": 0.95,
      "grad_norm": 22.927778244018555,
      "kl": 0.0,
      "learning_rate": 2.9516358463726884e-08,
      "logps/chosen": -323.7676696777344,
      "logps/rejected": -436.6930236816406,
      "loss": 0.1404,
      "rewards/chosen": 2.3460052013397217,
      "rewards/margins": 11.245654106140137,
      "rewards/rejected": -8.899648666381836,
      "step": 1480
    },
    {
      "epoch": 0.95,
      "grad_norm": 31.1018123626709,
      "kl": 0.0,
      "learning_rate": 2.7738264580369844e-08,
      "logps/chosen": -326.9073486328125,
      "logps/rejected": -446.97479248046875,
      "loss": 0.1572,
      "rewards/chosen": 2.2194085121154785,
      "rewards/margins": 12.011299133300781,
      "rewards/rejected": -9.791891098022461,
      "step": 1485
    },
    {
      "epoch": 0.95,
      "grad_norm": 27.92519760131836,
      "kl": 0.0,
      "learning_rate": 2.59601706970128e-08,
      "logps/chosen": -265.1518859863281,
      "logps/rejected": -455.7247009277344,
      "loss": 0.1584,
      "rewards/chosen": 2.3967678546905518,
      "rewards/margins": 11.583536148071289,
      "rewards/rejected": -9.186769485473633,
      "step": 1490
    },
    {
      "epoch": 0.96,
      "grad_norm": 21.29725456237793,
      "kl": 0.0,
      "learning_rate": 2.418207681365576e-08,
      "logps/chosen": -253.9468231201172,
      "logps/rejected": -426.5579528808594,
      "loss": 0.1479,
      "rewards/chosen": 2.272754192352295,
      "rewards/margins": 11.449048042297363,
      "rewards/rejected": -9.176294326782227,
      "step": 1495
    },
    {
      "epoch": 0.96,
      "grad_norm": 26.227853775024414,
      "kl": 0.0,
      "learning_rate": 2.240398293029872e-08,
      "logps/chosen": -270.14581298828125,
      "logps/rejected": -418.4717712402344,
      "loss": 0.1802,
      "rewards/chosen": 1.7642427682876587,
      "rewards/margins": 10.709261894226074,
      "rewards/rejected": -8.94502067565918,
      "step": 1500
    },
    {
      "epoch": 0.96,
      "grad_norm": 26.655635833740234,
      "kl": 0.0,
      "learning_rate": 2.0625889046941676e-08,
      "logps/chosen": -309.02117919921875,
      "logps/rejected": -432.2864685058594,
      "loss": 0.1593,
      "rewards/chosen": 2.1008925437927246,
      "rewards/margins": 11.279916763305664,
      "rewards/rejected": -9.179023742675781,
      "step": 1505
    },
    {
      "epoch": 0.97,
      "grad_norm": 25.60175323486328,
      "kl": 0.0,
      "learning_rate": 1.8847795163584636e-08,
      "logps/chosen": -268.1394958496094,
      "logps/rejected": -452.39202880859375,
      "loss": 0.1485,
      "rewards/chosen": 2.5142464637756348,
      "rewards/margins": 12.03423023223877,
      "rewards/rejected": -9.519984245300293,
      "step": 1510
    },
    {
      "epoch": 0.97,
      "grad_norm": 25.590343475341797,
      "kl": 0.0,
      "learning_rate": 1.7069701280227596e-08,
      "logps/chosen": -288.04571533203125,
      "logps/rejected": -439.8759765625,
      "loss": 0.162,
      "rewards/chosen": 2.3935296535491943,
      "rewards/margins": 11.7361421585083,
      "rewards/rejected": -9.342611312866211,
      "step": 1515
    },
    {
      "epoch": 0.97,
      "grad_norm": 19.91452407836914,
      "kl": 0.0,
      "learning_rate": 1.5291607396870554e-08,
      "logps/chosen": -288.63836669921875,
      "logps/rejected": -410.79815673828125,
      "loss": 0.1494,
      "rewards/chosen": 2.5123307704925537,
      "rewards/margins": 11.557938575744629,
      "rewards/rejected": -9.04560661315918,
      "step": 1520
    },
    {
      "epoch": 0.98,
      "grad_norm": 22.47270965576172,
      "kl": 0.0,
      "learning_rate": 1.3513513513513514e-08,
      "logps/chosen": -252.8772430419922,
      "logps/rejected": -407.09222412109375,
      "loss": 0.1353,
      "rewards/chosen": 2.3054208755493164,
      "rewards/margins": 11.360735893249512,
      "rewards/rejected": -9.055315017700195,
      "step": 1525
    },
    {
      "epoch": 0.98,
      "grad_norm": 24.753129959106445,
      "kl": 0.0,
      "learning_rate": 1.1735419630156473e-08,
      "logps/chosen": -282.2532958984375,
      "logps/rejected": -405.8521423339844,
      "loss": 0.1346,
      "rewards/chosen": 2.3494057655334473,
      "rewards/margins": 11.439948081970215,
      "rewards/rejected": -9.090542793273926,
      "step": 1530
    },
    {
      "epoch": 0.98,
      "grad_norm": 25.65488052368164,
      "kl": 0.0,
      "learning_rate": 9.95732574679943e-09,
      "logps/chosen": -289.83135986328125,
      "logps/rejected": -440.650634765625,
      "loss": 0.1592,
      "rewards/chosen": 2.2116000652313232,
      "rewards/margins": 11.503582954406738,
      "rewards/rejected": -9.291983604431152,
      "step": 1535
    },
    {
      "epoch": 0.99,
      "grad_norm": 28.50649070739746,
      "kl": 0.0,
      "learning_rate": 8.179231863442388e-09,
      "logps/chosen": -247.19186401367188,
      "logps/rejected": -430.9366149902344,
      "loss": 0.1613,
      "rewards/chosen": 2.0414962768554688,
      "rewards/margins": 11.605134963989258,
      "rewards/rejected": -9.563637733459473,
      "step": 1540
    },
    {
      "epoch": 0.99,
      "grad_norm": 22.502891540527344,
      "kl": 0.0,
      "learning_rate": 6.401137980085348e-09,
      "logps/chosen": -280.7741394042969,
      "logps/rejected": -432.50140380859375,
      "loss": 0.1613,
      "rewards/chosen": 2.1499953269958496,
      "rewards/margins": 10.987812995910645,
      "rewards/rejected": -8.837817192077637,
      "step": 1545
    },
    {
      "epoch": 0.99,
      "grad_norm": 38.24708938598633,
      "kl": 0.0,
      "learning_rate": 4.623044096728307e-09,
      "logps/chosen": -268.7917785644531,
      "logps/rejected": -439.6647033691406,
      "loss": 0.1653,
      "rewards/chosen": 2.114647388458252,
      "rewards/margins": 11.40100383758545,
      "rewards/rejected": -9.286355972290039,
      "step": 1550
    },
    {
      "epoch": 0.99,
      "grad_norm": 28.025665283203125,
      "kl": 0.0,
      "learning_rate": 2.844950213371266e-09,
      "logps/chosen": -299.0115051269531,
      "logps/rejected": -446.5692443847656,
      "loss": 0.1523,
      "rewards/chosen": 2.446258783340454,
      "rewards/margins": 11.819780349731445,
      "rewards/rejected": -9.373520851135254,
      "step": 1555
    },
    {
      "epoch": 1.0,
      "grad_norm": 31.7341251373291,
      "kl": 0.0,
      "learning_rate": 1.0668563300142248e-09,
      "logps/chosen": -234.8123779296875,
      "logps/rejected": -408.6230773925781,
      "loss": 0.1703,
      "rewards/chosen": 2.086718797683716,
      "rewards/margins": 11.345036506652832,
      "rewards/rejected": -9.258317947387695,
      "step": 1560
    },
    {
      "epoch": 1.0,
      "step": 1563,
      "total_flos": 0.0,
      "train_loss": 0.2182784026735346,
      "train_runtime": 11066.0236,
      "train_samples_per_second": 9.037,
      "train_steps_per_second": 0.141
    }
  ],
  "logging_steps": 5,
  "max_steps": 1563,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}