{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9952,
  "eval_steps": 500,
  "global_step": 936,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.016,
      "grad_norm": 38.78816510379247,
      "learning_rate": 2.6595744680851066e-07,
      "logits/chosen": -0.9742305874824524,
      "logits/rejected": -1.0986480712890625,
      "logps/chosen": -1.3971922397613525,
      "logps/rejected": -2.0653083324432373,
      "loss": 1.7192,
      "odds_ratio_loss": 0.5222790241241455,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06985961645841599,
      "rewards/margins": 0.03340579569339752,
      "rewards/rejected": -0.1032654196023941,
      "sft_loss": 1.3971922397613525,
      "step": 5
    },
    {
      "epoch": 0.032,
      "grad_norm": 34.938822021789555,
      "learning_rate": 5.319148936170213e-07,
      "logits/chosen": -0.9504960775375366,
      "logits/rejected": -0.9644749760627747,
      "logps/chosen": -1.5183446407318115,
      "logps/rejected": -1.7423614263534546,
      "loss": 1.7,
      "odds_ratio_loss": 0.6908475756645203,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07591722905635834,
      "rewards/margins": 0.011200849898159504,
      "rewards/rejected": -0.08711807429790497,
      "sft_loss": 1.5183446407318115,
      "step": 10
    },
    {
      "epoch": 0.048,
      "grad_norm": 16.339902176867668,
      "learning_rate": 7.97872340425532e-07,
      "logits/chosen": -0.9059197306632996,
      "logits/rejected": -0.9509723782539368,
      "logps/chosen": -1.3207223415374756,
      "logps/rejected": -2.0547330379486084,
      "loss": 1.6089,
      "odds_ratio_loss": 0.6031222343444824,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06603612005710602,
      "rewards/margins": 0.036700539290905,
      "rewards/rejected": -0.10273666679859161,
      "sft_loss": 1.3207223415374756,
      "step": 15
    },
    {
      "epoch": 0.064,
      "grad_norm": 20.917533328584984,
      "learning_rate": 1.0638297872340427e-06,
      "logits/chosen": -0.9647526741027832,
      "logits/rejected": -1.0629401206970215,
      "logps/chosen": -1.4611456394195557,
      "logps/rejected": -2.1091372966766357,
      "loss": 1.4991,
      "odds_ratio_loss": 0.5197792649269104,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07305728644132614,
      "rewards/margins": 0.03239959105849266,
      "rewards/rejected": -0.1054568737745285,
      "sft_loss": 1.4611456394195557,
      "step": 20
    },
    {
      "epoch": 0.08,
      "grad_norm": 18.664826990233166,
      "learning_rate": 1.3297872340425533e-06,
      "logits/chosen": -1.054176688194275,
      "logits/rejected": -1.3495676517486572,
      "logps/chosen": -1.3905874490737915,
      "logps/rejected": -1.7380883693695068,
      "loss": 1.3634,
      "odds_ratio_loss": 0.5891755819320679,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06952936947345734,
      "rewards/margins": 0.017375053837895393,
      "rewards/rejected": -0.08690442144870758,
      "sft_loss": 1.3905874490737915,
      "step": 25
    },
    {
      "epoch": 0.096,
      "grad_norm": 10.604261736039259,
      "learning_rate": 1.595744680851064e-06,
      "logits/chosen": -0.9233123660087585,
      "logits/rejected": -0.9947792887687683,
      "logps/chosen": -1.3364614248275757,
      "logps/rejected": -1.4293016195297241,
      "loss": 1.3477,
      "odds_ratio_loss": 0.7917504906654358,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0668230801820755,
      "rewards/margins": 0.0046419971622526646,
      "rewards/rejected": -0.07146507501602173,
      "sft_loss": 1.3364614248275757,
      "step": 30
    },
    {
      "epoch": 0.112,
      "grad_norm": 9.23627597385138,
      "learning_rate": 1.8617021276595745e-06,
      "logits/chosen": -1.1354405879974365,
      "logits/rejected": -1.1504920721054077,
      "logps/chosen": -1.2876451015472412,
      "logps/rejected": -1.394587755203247,
      "loss": 1.3323,
      "odds_ratio_loss": 0.7012001276016235,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06438224762678146,
      "rewards/margins": 0.005347140599042177,
      "rewards/rejected": -0.06972938776016235,
      "sft_loss": 1.2876451015472412,
      "step": 35
    },
    {
      "epoch": 0.128,
      "grad_norm": 10.737738874884705,
      "learning_rate": 2.1276595744680853e-06,
      "logits/chosen": -1.0701355934143066,
      "logits/rejected": -1.2617167234420776,
      "logps/chosen": -1.345517873764038,
      "logps/rejected": -1.5351002216339111,
      "loss": 1.3194,
      "odds_ratio_loss": 0.6834388971328735,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06727589666843414,
      "rewards/margins": 0.009479111060500145,
      "rewards/rejected": -0.07675500959157944,
      "sft_loss": 1.345517873764038,
      "step": 40
    },
    {
      "epoch": 0.144,
      "grad_norm": 3.92140979781429,
      "learning_rate": 2.393617021276596e-06,
      "logits/chosen": -0.8892070651054382,
      "logits/rejected": -0.9303566217422485,
      "logps/chosen": -1.1515204906463623,
      "logps/rejected": -1.311240315437317,
      "loss": 1.3284,
      "odds_ratio_loss": 0.6429694890975952,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05757603049278259,
      "rewards/margins": 0.007985993288457394,
      "rewards/rejected": -0.06556202471256256,
      "sft_loss": 1.1515204906463623,
      "step": 45
    },
    {
      "epoch": 0.16,
      "grad_norm": 12.131879118457386,
      "learning_rate": 2.6595744680851065e-06,
      "logits/chosen": -0.9589262008666992,
      "logits/rejected": -1.0993459224700928,
      "logps/chosen": -1.3476245403289795,
      "logps/rejected": -1.4529037475585938,
      "loss": 1.3627,
      "odds_ratio_loss": 0.6757664680480957,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06738122552633286,
      "rewards/margins": 0.005263959523290396,
      "rewards/rejected": -0.07264517992734909,
      "sft_loss": 1.3476245403289795,
      "step": 50
    },
    {
      "epoch": 0.176,
      "grad_norm": 6.920195139152275,
      "learning_rate": 2.9255319148936174e-06,
      "logits/chosen": -0.9547260403633118,
      "logits/rejected": -1.137032151222229,
      "logps/chosen": -1.3528764247894287,
      "logps/rejected": -1.5240767002105713,
      "loss": 1.2606,
      "odds_ratio_loss": 0.6684664487838745,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06764382123947144,
      "rewards/margins": 0.008560018613934517,
      "rewards/rejected": -0.0762038379907608,
      "sft_loss": 1.3528764247894287,
      "step": 55
    },
    {
      "epoch": 0.192,
      "grad_norm": 5.7098162603211104,
      "learning_rate": 3.191489361702128e-06,
      "logits/chosen": -0.982739269733429,
      "logits/rejected": -1.1074873208999634,
      "logps/chosen": -1.4583510160446167,
      "logps/rejected": -1.5452958345413208,
      "loss": 1.3274,
      "odds_ratio_loss": 0.7071143388748169,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.07291755080223083,
      "rewards/margins": 0.00434724148362875,
      "rewards/rejected": -0.07726480066776276,
      "sft_loss": 1.4583510160446167,
      "step": 60
    },
    {
      "epoch": 0.208,
      "grad_norm": 4.824404122964849,
      "learning_rate": 3.457446808510639e-06,
      "logits/chosen": -0.9817106127738953,
      "logits/rejected": -1.111875295639038,
      "logps/chosen": -1.3345075845718384,
      "logps/rejected": -1.5089738368988037,
      "loss": 1.2879,
      "odds_ratio_loss": 0.6608881950378418,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06672537326812744,
      "rewards/margins": 0.008723323233425617,
      "rewards/rejected": -0.07544869929552078,
      "sft_loss": 1.3345075845718384,
      "step": 65
    },
    {
      "epoch": 0.224,
      "grad_norm": 6.021701349784469,
      "learning_rate": 3.723404255319149e-06,
      "logits/chosen": -1.1069040298461914,
      "logits/rejected": -1.075409173965454,
      "logps/chosen": -1.2757076025009155,
      "logps/rejected": -1.4533952474594116,
      "loss": 1.3134,
      "odds_ratio_loss": 0.7258908152580261,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0637853816151619,
      "rewards/margins": 0.008884383365511894,
      "rewards/rejected": -0.07266975939273834,
      "sft_loss": 1.2757076025009155,
      "step": 70
    },
    {
      "epoch": 0.24,
      "grad_norm": 5.02350299089495,
      "learning_rate": 3.98936170212766e-06,
      "logits/chosen": -0.9449170231819153,
      "logits/rejected": -1.1291000843048096,
      "logps/chosen": -1.2914139032363892,
      "logps/rejected": -1.484882116317749,
      "loss": 1.2797,
      "odds_ratio_loss": 0.6280355453491211,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06457068026065826,
      "rewards/margins": 0.009673424996435642,
      "rewards/rejected": -0.07424411177635193,
      "sft_loss": 1.2914139032363892,
      "step": 75
    },
    {
      "epoch": 0.256,
      "grad_norm": 5.308377873815178,
      "learning_rate": 4.255319148936171e-06,
      "logits/chosen": -0.9751418828964233,
      "logits/rejected": -1.094626545906067,
      "logps/chosen": -1.0752859115600586,
      "logps/rejected": -1.334636926651001,
      "loss": 1.2678,
      "odds_ratio_loss": 0.5908417701721191,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05376429483294487,
      "rewards/margins": 0.012967551127076149,
      "rewards/rejected": -0.06673184782266617,
      "sft_loss": 1.0752859115600586,
      "step": 80
    },
    {
      "epoch": 0.272,
      "grad_norm": 6.346400541372991,
      "learning_rate": 4.521276595744681e-06,
      "logits/chosen": -0.9909063577651978,
      "logits/rejected": -1.0453189611434937,
      "logps/chosen": -1.0384023189544678,
      "logps/rejected": -1.2738986015319824,
      "loss": 1.2615,
      "odds_ratio_loss": 0.6149815320968628,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.051920123398303986,
      "rewards/margins": 0.011774817481637001,
      "rewards/rejected": -0.06369493901729584,
      "sft_loss": 1.0384023189544678,
      "step": 85
    },
    {
      "epoch": 0.288,
      "grad_norm": 6.055157418934933,
      "learning_rate": 4.787234042553192e-06,
      "logits/chosen": -0.9102107882499695,
      "logits/rejected": -0.9935488700866699,
      "logps/chosen": -1.1961522102355957,
      "logps/rejected": -1.415398359298706,
      "loss": 1.212,
      "odds_ratio_loss": 0.6041514277458191,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.059807609766721725,
      "rewards/margins": 0.010962305590510368,
      "rewards/rejected": -0.07076992094516754,
      "sft_loss": 1.1961522102355957,
      "step": 90
    },
    {
      "epoch": 0.304,
      "grad_norm": 5.552038237166853,
      "learning_rate": 4.999982598564682e-06,
      "logits/chosen": -0.9868474006652832,
      "logits/rejected": -1.0573745965957642,
      "logps/chosen": -1.2718483209609985,
      "logps/rejected": -1.5264861583709717,
      "loss": 1.2618,
      "odds_ratio_loss": 0.640211820602417,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06359241157770157,
      "rewards/margins": 0.012731892988085747,
      "rewards/rejected": -0.07632430642843246,
      "sft_loss": 1.2718483209609985,
      "step": 95
    },
    {
      "epoch": 0.32,
      "grad_norm": 7.727039615086689,
      "learning_rate": 4.999373573764188e-06,
      "logits/chosen": -1.0175714492797852,
      "logits/rejected": -0.8734873533248901,
      "logps/chosen": -1.199947476387024,
      "logps/rejected": -1.3050801753997803,
      "loss": 1.2807,
      "odds_ratio_loss": 0.7060422897338867,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.059997379779815674,
      "rewards/margins": 0.005256640259176493,
      "rewards/rejected": -0.06525401026010513,
      "sft_loss": 1.199947476387024,
      "step": 100
    },
    {
      "epoch": 0.336,
      "grad_norm": 6.365580704740442,
      "learning_rate": 4.997894719430564e-06,
      "logits/chosen": -0.8132748603820801,
      "logits/rejected": -0.9986258745193481,
      "logps/chosen": -1.1854135990142822,
      "logps/rejected": -1.5025197267532349,
      "loss": 1.2805,
      "odds_ratio_loss": 0.6134302020072937,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05927068740129471,
      "rewards/margins": 0.015855297446250916,
      "rewards/rejected": -0.07512599229812622,
      "sft_loss": 1.1854135990142822,
      "step": 105
    },
    {
      "epoch": 0.352,
      "grad_norm": 6.860280491199452,
      "learning_rate": 4.995546550233241e-06,
      "logits/chosen": -0.9532275199890137,
      "logits/rejected": -1.136918306350708,
      "logps/chosen": -1.408894658088684,
      "logps/rejected": -1.558379888534546,
      "loss": 1.2769,
      "odds_ratio_loss": 0.7106048464775085,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0704447329044342,
      "rewards/margins": 0.007474270649254322,
      "rewards/rejected": -0.07791899144649506,
      "sft_loss": 1.408894658088684,
      "step": 110
    },
    {
      "epoch": 0.368,
      "grad_norm": 5.854849184237901,
      "learning_rate": 4.992329883379755e-06,
      "logits/chosen": -0.9923961758613586,
      "logits/rejected": -1.2076470851898193,
      "logps/chosen": -1.4104652404785156,
      "logps/rejected": -1.4293034076690674,
      "loss": 1.2514,
      "odds_ratio_loss": 0.7473000288009644,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07052326202392578,
      "rewards/margins": 0.0009419171256013215,
      "rewards/rejected": -0.07146517932415009,
      "sft_loss": 1.4104652404785156,
      "step": 115
    },
    {
      "epoch": 0.384,
      "grad_norm": 4.334718029265899,
      "learning_rate": 4.988245838331339e-06,
      "logits/chosen": -0.9599634408950806,
      "logits/rejected": -0.9879687428474426,
      "logps/chosen": -1.2545793056488037,
      "logps/rejected": -1.4001166820526123,
      "loss": 1.2204,
      "odds_ratio_loss": 0.6919530630111694,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.06272897124290466,
      "rewards/margins": 0.007276860065758228,
      "rewards/rejected": -0.07000583410263062,
      "sft_loss": 1.2545793056488037,
      "step": 120
    },
    {
      "epoch": 0.4,
      "grad_norm": 6.640886415637415,
      "learning_rate": 4.983295836413337e-06,
      "logits/chosen": -0.8892742395401001,
      "logits/rejected": -0.8854384422302246,
      "logps/chosen": -1.1642634868621826,
      "logps/rejected": -1.2482268810272217,
      "loss": 1.225,
      "odds_ratio_loss": 0.7551355361938477,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05821318179368973,
      "rewards/margins": 0.004198164213448763,
      "rewards/rejected": -0.0624113492667675,
      "sft_loss": 1.1642634868621826,
      "step": 125
    },
    {
      "epoch": 0.416,
      "grad_norm": 3.3977384471741727,
      "learning_rate": 4.977481600320545e-06,
      "logits/chosen": -1.0043504238128662,
      "logits/rejected": -1.0277382135391235,
      "logps/chosen": -1.2277835607528687,
      "logps/rejected": -1.3244746923446655,
      "loss": 1.2696,
      "odds_ratio_loss": 0.726096510887146,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06138917803764343,
      "rewards/margins": 0.0048345597460865974,
      "rewards/rejected": -0.06622373312711716,
      "sft_loss": 1.2277835607528687,
      "step": 130
    },
    {
      "epoch": 0.432,
      "grad_norm": 29.84907862640638,
      "learning_rate": 4.970805153517692e-06,
      "logits/chosen": -0.906798243522644,
      "logits/rejected": -1.0836145877838135,
      "logps/chosen": -1.273900032043457,
      "logps/rejected": -1.6986215114593506,
      "loss": 1.2562,
      "odds_ratio_loss": 0.6722120642662048,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06369500607252121,
      "rewards/margins": 0.02123607136309147,
      "rewards/rejected": -0.08493108302354813,
      "sft_loss": 1.273900032043457,
      "step": 135
    },
    {
      "epoch": 0.448,
      "grad_norm": 9.009223423852758,
      "learning_rate": 4.963268819535228e-06,
      "logits/chosen": -0.9239026308059692,
      "logits/rejected": -0.9300885200500488,
      "logps/chosen": -1.0778651237487793,
      "logps/rejected": -1.2015340328216553,
      "loss": 1.2297,
      "odds_ratio_loss": 0.6974226832389832,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.053893256932497025,
      "rewards/margins": 0.006183440797030926,
      "rewards/rejected": -0.06007670238614082,
      "sft_loss": 1.0778651237487793,
      "step": 140
    },
    {
      "epoch": 0.464,
      "grad_norm": 6.881851804175557,
      "learning_rate": 4.954875221160695e-06,
      "logits/chosen": -0.9062795639038086,
      "logits/rejected": -0.9437534213066101,
      "logps/chosen": -1.151301622390747,
      "logps/rejected": -1.376366376876831,
      "loss": 1.1857,
      "odds_ratio_loss": 0.6390897035598755,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05756508558988571,
      "rewards/margins": 0.01125323586165905,
      "rewards/rejected": -0.06881832331418991,
      "sft_loss": 1.151301622390747,
      "step": 145
    },
    {
      "epoch": 0.48,
      "grad_norm": 6.024680716397588,
      "learning_rate": 4.945627279525943e-06,
      "logits/chosen": -0.8047590255737305,
      "logits/rejected": -1.0275001525878906,
      "logps/chosen": -1.1844927072525024,
      "logps/rejected": -1.1749188899993896,
      "loss": 1.2559,
      "odds_ratio_loss": 0.7716752886772156,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.05922463536262512,
      "rewards/margins": -0.0004786819336004555,
      "rewards/rejected": -0.05874595046043396,
      "sft_loss": 1.1844927072525024,
      "step": 150
    },
    {
      "epoch": 0.496,
      "grad_norm": 4.969594803216186,
      "learning_rate": 4.935528213090526e-06,
      "logits/chosen": -0.8657525181770325,
      "logits/rejected": -0.9782077074050903,
      "logps/chosen": -1.2747188806533813,
      "logps/rejected": -1.331703543663025,
      "loss": 1.2301,
      "odds_ratio_loss": 0.720513641834259,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.06373593956232071,
      "rewards/margins": 0.0028492447454482317,
      "rewards/rejected": -0.06658518314361572,
      "sft_loss": 1.2747188806533813,
      "step": 155
    },
    {
      "epoch": 0.512,
      "grad_norm": 5.691340876461069,
      "learning_rate": 4.9245815365216115e-06,
      "logits/chosen": -0.7511823773384094,
      "logits/rejected": -1.0508475303649902,
      "logps/chosen": -1.1776434183120728,
      "logps/rejected": -1.6007537841796875,
      "loss": 1.221,
      "odds_ratio_loss": 0.5590518116950989,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05888216570019722,
      "rewards/margins": 0.021155523136258125,
      "rewards/rejected": -0.08003769814968109,
      "sft_loss": 1.1776434183120728,
      "step": 160
    },
    {
      "epoch": 0.528,
      "grad_norm": 6.0040479808042235,
      "learning_rate": 4.912791059470815e-06,
      "logits/chosen": -0.9242817163467407,
      "logits/rejected": -0.8517144918441772,
      "logps/chosen": -1.0615520477294922,
      "logps/rejected": -1.288195252418518,
      "loss": 1.1745,
      "odds_ratio_loss": 0.6426531076431274,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05307760834693909,
      "rewards/margins": 0.011332156136631966,
      "rewards/rejected": -0.0644097551703453,
      "sft_loss": 1.0615520477294922,
      "step": 165
    },
    {
      "epoch": 0.544,
      "grad_norm": 5.529927712259793,
      "learning_rate": 4.900160885248363e-06,
      "logits/chosen": -0.9031238555908203,
      "logits/rejected": -0.8960297703742981,
      "logps/chosen": -1.2523874044418335,
      "logps/rejected": -1.4036993980407715,
      "loss": 1.194,
      "odds_ratio_loss": 0.6845428347587585,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06261937320232391,
      "rewards/margins": 0.007565596606582403,
      "rewards/rejected": -0.07018496841192245,
      "sft_loss": 1.2523874044418335,
      "step": 170
    },
    {
      "epoch": 0.56,
      "grad_norm": 4.1316911858664644,
      "learning_rate": 4.886695409395068e-06,
      "logits/chosen": -0.853635311126709,
      "logits/rejected": -1.0007998943328857,
      "logps/chosen": -1.158414602279663,
      "logps/rejected": -1.1915799379348755,
      "loss": 1.2303,
      "odds_ratio_loss": 0.7887060046195984,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.05792072415351868,
      "rewards/margins": 0.0016582731623202562,
      "rewards/rejected": -0.059578996151685715,
      "sft_loss": 1.158414602279663,
      "step": 175
    },
    {
      "epoch": 0.576,
      "grad_norm": 4.185731700055316,
      "learning_rate": 4.872399318152594e-06,
      "logits/chosen": -0.9244221448898315,
      "logits/rejected": -0.8243404626846313,
      "logps/chosen": -1.124448537826538,
      "logps/rejected": -1.1855733394622803,
      "loss": 1.2359,
      "odds_ratio_loss": 0.6966110467910767,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.05622243881225586,
      "rewards/margins": 0.0030562314204871655,
      "rewards/rejected": -0.059278666973114014,
      "sft_loss": 1.124448537826538,
      "step": 180
    },
    {
      "epoch": 0.592,
      "grad_norm": 5.534373355221194,
      "learning_rate": 4.857277586832556e-06,
      "logits/chosen": -0.8700960278511047,
      "logits/rejected": -1.0795601606369019,
      "logps/chosen": -1.1355112791061401,
      "logps/rejected": -1.3763678073883057,
      "loss": 1.2299,
      "odds_ratio_loss": 0.61272132396698,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.05677555873990059,
      "rewards/margins": 0.012042825110256672,
      "rewards/rejected": -0.06881839036941528,
      "sft_loss": 1.1355112791061401,
      "step": 185
    },
    {
      "epoch": 0.608,
      "grad_norm": 8.320294434791975,
      "learning_rate": 4.841335478085015e-06,
      "logits/chosen": -0.837981104850769,
      "logits/rejected": -1.041286587715149,
      "logps/chosen": -1.0703322887420654,
      "logps/rejected": -1.6396074295043945,
      "loss": 1.1951,
      "odds_ratio_loss": 0.5467146635055542,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05351661518216133,
      "rewards/margins": 0.028463756665587425,
      "rewards/rejected": -0.08198036998510361,
      "sft_loss": 1.0703322887420654,
      "step": 190
    },
    {
      "epoch": 0.624,
      "grad_norm": 4.60123509876027,
      "learning_rate": 4.824578540066981e-06,
      "logits/chosen": -0.93339604139328,
      "logits/rejected": -0.8643118739128113,
      "logps/chosen": -1.3019145727157593,
      "logps/rejected": -1.2324789762496948,
      "loss": 1.2285,
      "odds_ratio_loss": 0.8142224550247192,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.06509573012590408,
      "rewards/margins": -0.003471782896667719,
      "rewards/rejected": -0.061623942106962204,
      "sft_loss": 1.3019145727157593,
      "step": 195
    },
    {
      "epoch": 0.64,
      "grad_norm": 21.66363438026074,
      "learning_rate": 4.807012604511542e-06,
      "logits/chosen": -0.6915689706802368,
      "logits/rejected": -0.897596538066864,
      "logps/chosen": -1.1985743045806885,
      "logps/rejected": -1.3054250478744507,
      "loss": 1.2092,
      "odds_ratio_loss": 0.6641479730606079,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.059928715229034424,
      "rewards/margins": 0.005342531017959118,
      "rewards/rejected": -0.06527124345302582,
      "sft_loss": 1.1985743045806885,
      "step": 200
    },
    {
      "epoch": 0.656,
      "grad_norm": 5.945812709802962,
      "learning_rate": 4.788643784698316e-06,
      "logits/chosen": -0.8415448069572449,
      "logits/rejected": -0.8126734495162964,
      "logps/chosen": -1.1990188360214233,
      "logps/rejected": -1.1923567056655884,
      "loss": 1.1919,
      "odds_ratio_loss": 0.7749138474464417,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05995094031095505,
      "rewards/margins": -0.0003331051266286522,
      "rewards/rejected": -0.05961783602833748,
      "sft_loss": 1.1990188360214233,
      "step": 205
    },
    {
      "epoch": 0.672,
      "grad_norm": 6.175015091799187,
      "learning_rate": 4.769478473325908e-06,
      "logits/chosen": -0.9676336050033569,
      "logits/rejected": -1.0954523086547852,
      "logps/chosen": -1.0937362909317017,
      "logps/rejected": -1.169339895248413,
      "loss": 1.2244,
      "odds_ratio_loss": 0.7255644798278809,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.054686807096004486,
      "rewards/margins": 0.003780178725719452,
      "rewards/rejected": -0.058466989547014236,
      "sft_loss": 1.0937362909317017,
      "step": 210
    },
    {
      "epoch": 0.688,
      "grad_norm": 4.1222127125265935,
      "learning_rate": 4.7495233402871336e-06,
      "logits/chosen": -0.7574236392974854,
      "logits/rejected": -0.9107638597488403,
      "logps/chosen": -1.0908468961715698,
      "logps/rejected": -1.3144160509109497,
      "loss": 1.2612,
      "odds_ratio_loss": 0.6122642159461975,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05454235151410103,
      "rewards/margins": 0.011178453452885151,
      "rewards/rejected": -0.0657208114862442,
      "sft_loss": 1.0908468961715698,
      "step": 215
    },
    {
      "epoch": 0.704,
      "grad_norm": 6.629592523219295,
      "learning_rate": 4.728785330347771e-06,
      "logits/chosen": -1.0126041173934937,
      "logits/rejected": -1.1902836561203003,
      "logps/chosen": -1.0830891132354736,
      "logps/rejected": -1.2832982540130615,
      "loss": 1.1849,
      "odds_ratio_loss": 0.6457486748695374,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05415446311235428,
      "rewards/margins": 0.010010452009737492,
      "rewards/rejected": -0.0641649141907692,
      "sft_loss": 1.0830891132354736,
      "step": 220
    },
    {
      "epoch": 0.72,
      "grad_norm": 4.769856705836216,
      "learning_rate": 4.70727166072964e-06,
      "logits/chosen": -0.9934102296829224,
      "logits/rejected": -1.0441877841949463,
      "logps/chosen": -1.1385221481323242,
      "logps/rejected": -1.3962942361831665,
      "loss": 1.2004,
      "odds_ratio_loss": 0.631730318069458,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.05692611262202263,
      "rewards/margins": 0.012888607569038868,
      "rewards/rejected": -0.06981472671031952,
      "sft_loss": 1.1385221481323242,
      "step": 225
    },
    {
      "epoch": 0.736,
      "grad_norm": 9.602754215145033,
      "learning_rate": 4.684989818598887e-06,
      "logits/chosen": -0.8686431050300598,
      "logits/rejected": -0.9307680130004883,
      "logps/chosen": -1.2431721687316895,
      "logps/rejected": -1.3505405187606812,
      "loss": 1.1679,
      "odds_ratio_loss": 0.7377614974975586,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06215860694646835,
      "rewards/margins": 0.005368414800614119,
      "rewards/rejected": -0.06752702593803406,
      "sft_loss": 1.2431721687316895,
      "step": 230
    },
    {
      "epoch": 0.752,
      "grad_norm": 4.991520073276977,
      "learning_rate": 4.661947558460296e-06,
      "logits/chosen": -0.893735408782959,
      "logits/rejected": -1.037488579750061,
      "logps/chosen": -1.2366321086883545,
      "logps/rejected": -1.422572135925293,
      "loss": 1.1988,
      "odds_ratio_loss": 0.6442903280258179,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06183161213994026,
      "rewards/margins": 0.009297001175582409,
      "rewards/rejected": -0.07112861424684525,
      "sft_loss": 1.2366321086883545,
      "step": 235
    },
    {
      "epoch": 0.768,
      "grad_norm": 5.765863831511136,
      "learning_rate": 4.63815289945858e-06,
      "logits/chosen": -0.8988453149795532,
      "logits/rejected": -1.1190879344940186,
      "logps/chosen": -1.1124114990234375,
      "logps/rejected": -1.2776150703430176,
      "loss": 1.2519,
      "odds_ratio_loss": 0.697569727897644,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.055620573461055756,
      "rewards/margins": 0.008260180242359638,
      "rewards/rejected": -0.06388075649738312,
      "sft_loss": 1.1124114990234375,
      "step": 240
    },
    {
      "epoch": 0.784,
      "grad_norm": 4.2480178226924465,
      "learning_rate": 4.613614122587563e-06,
      "logits/chosen": -0.8890430331230164,
      "logits/rejected": -0.993693470954895,
      "logps/chosen": -1.1632936000823975,
      "logps/rejected": -1.2893292903900146,
      "loss": 1.2055,
      "odds_ratio_loss": 0.7878369092941284,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.058164678514003754,
      "rewards/margins": 0.0063017881475389,
      "rewards/rejected": -0.0644664615392685,
      "sft_loss": 1.1632936000823975,
      "step": 245
    },
    {
      "epoch": 0.8,
      "grad_norm": 4.066953723548615,
      "learning_rate": 4.5883397678082385e-06,
      "logits/chosen": -0.8833068013191223,
      "logits/rejected": -1.0138885974884033,
      "logps/chosen": -1.2117881774902344,
      "logps/rejected": -1.1593796014785767,
      "loss": 1.1903,
      "odds_ratio_loss": 0.7823435664176941,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.06058941036462784,
      "rewards/margins": -0.002620431361719966,
      "rewards/rejected": -0.05796898156404495,
      "sft_loss": 1.2117881774902344,
      "step": 250
    },
    {
      "epoch": 0.816,
      "grad_norm": 5.662450247054175,
      "learning_rate": 4.562338631076703e-06,
      "logits/chosen": -1.0948327779769897,
      "logits/rejected": -1.1347147226333618,
      "logps/chosen": -1.0046937465667725,
      "logps/rejected": -1.2093970775604248,
      "loss": 1.1825,
      "odds_ratio_loss": 0.6374476552009583,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.05023468658328056,
      "rewards/margins": 0.010235178284347057,
      "rewards/rejected": -0.060469865798950195,
      "sft_loss": 1.0046937465667725,
      "step": 255
    },
    {
      "epoch": 0.832,
      "grad_norm": 5.116135672128333,
      "learning_rate": 4.535619761282989e-06,
      "logits/chosen": -0.9458154439926147,
      "logits/rejected": -1.224017858505249,
      "logps/chosen": -1.202074646949768,
      "logps/rejected": -1.3588545322418213,
      "loss": 1.23,
      "odds_ratio_loss": 0.668530285358429,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06010373681783676,
      "rewards/margins": 0.007838994264602661,
      "rewards/rejected": -0.06794272363185883,
      "sft_loss": 1.202074646949768,
      "step": 260
    },
    {
      "epoch": 0.848,
      "grad_norm": 8.619415579204428,
      "learning_rate": 4.508192457101886e-06,
      "logits/chosen": -1.0687763690948486,
      "logits/rejected": -1.213895320892334,
      "logps/chosen": -1.2021548748016357,
      "logps/rejected": -1.541288137435913,
      "loss": 1.2482,
      "odds_ratio_loss": 0.658495306968689,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.060107748955488205,
      "rewards/margins": 0.016956666484475136,
      "rewards/rejected": -0.0770644098520279,
      "sft_loss": 1.2021548748016357,
      "step": 265
    },
    {
      "epoch": 0.864,
      "grad_norm": 5.000713262682842,
      "learning_rate": 4.480066263756821e-06,
      "logits/chosen": -0.9636927843093872,
      "logits/rejected": -1.1979575157165527,
      "logps/chosen": -1.220902442932129,
      "logps/rejected": -1.5320258140563965,
      "loss": 1.1889,
      "odds_ratio_loss": 0.5880663990974426,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06104512885212898,
      "rewards/margins": 0.01555616408586502,
      "rewards/rejected": -0.0766012892127037,
      "sft_loss": 1.220902442932129,
      "step": 270
    },
    {
      "epoch": 0.88,
      "grad_norm": 6.295390583446683,
      "learning_rate": 4.451250969697944e-06,
      "logits/chosen": -0.9099109768867493,
      "logits/rejected": -1.1926413774490356,
      "logps/chosen": -1.1539406776428223,
      "logps/rejected": -1.4602700471878052,
      "loss": 1.2078,
      "odds_ratio_loss": 0.5688539743423462,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05769703909754753,
      "rewards/margins": 0.015316471457481384,
      "rewards/rejected": -0.07301349937915802,
      "sft_loss": 1.1539406776428223,
      "step": 275
    },
    {
      "epoch": 0.896,
      "grad_norm": 3.602560879317352,
      "learning_rate": 4.42175660319555e-06,
      "logits/chosen": -1.159501314163208,
      "logits/rejected": -1.2406375408172607,
      "logps/chosen": -1.192801594734192,
      "logps/rejected": -1.3772008419036865,
      "loss": 1.1898,
      "odds_ratio_loss": 0.7453117966651917,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.059640079736709595,
      "rewards/margins": 0.00921996496617794,
      "rewards/rejected": -0.06886004656553268,
      "sft_loss": 1.192801594734192,
      "step": 280
    },
    {
      "epoch": 0.912,
      "grad_norm": 8.783987785350908,
      "learning_rate": 4.391593428850069e-06,
      "logits/chosen": -0.9893490076065063,
      "logits/rejected": -1.1613223552703857,
      "logps/chosen": -1.0789204835891724,
      "logps/rejected": -1.1841206550598145,
      "loss": 1.2078,
      "odds_ratio_loss": 0.7538634538650513,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05394602566957474,
      "rewards/margins": 0.005260012112557888,
      "rewards/rejected": -0.0592060312628746,
      "sft_loss": 1.0789204835891724,
      "step": 285
    },
    {
      "epoch": 0.928,
      "grad_norm": 6.736280205252853,
      "learning_rate": 4.360771944019767e-06,
      "logits/chosen": -1.0662925243377686,
      "logits/rejected": -1.1013010740280151,
      "logps/chosen": -1.2803829908370972,
      "logps/rejected": -1.6290887594223022,
      "loss": 1.2225,
      "odds_ratio_loss": 0.6528395414352417,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06401915848255157,
      "rewards/margins": 0.017435286194086075,
      "rewards/rejected": -0.08145444095134735,
      "sft_loss": 1.2803829908370972,
      "step": 290
    },
    {
      "epoch": 0.944,
      "grad_norm": 7.2196796373916365,
      "learning_rate": 4.329302875167486e-06,
      "logits/chosen": -1.0543363094329834,
      "logits/rejected": -1.2899835109710693,
      "logps/chosen": -1.1794860363006592,
      "logps/rejected": -1.4053813219070435,
      "loss": 1.2094,
      "odds_ratio_loss": 0.6699637174606323,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05897430330514908,
      "rewards/margins": 0.011294771917164326,
      "rewards/rejected": -0.07026907801628113,
      "sft_loss": 1.1794860363006592,
      "step": 295
    },
    {
      "epoch": 0.96,
      "grad_norm": 5.962697902517566,
      "learning_rate": 4.297197174127619e-06,
      "logits/chosen": -1.0228321552276611,
      "logits/rejected": -1.2345274686813354,
      "logps/chosen": -1.0752114057540894,
      "logps/rejected": -1.2755658626556396,
      "loss": 1.212,
      "odds_ratio_loss": 0.6680848002433777,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05376056954264641,
      "rewards/margins": 0.01001772377640009,
      "rewards/rejected": -0.06377829611301422,
      "sft_loss": 1.0752114057540894,
      "step": 300
    },
    {
      "epoch": 0.976,
      "grad_norm": 3.4092575585850917,
      "learning_rate": 4.2644660142946685e-06,
      "logits/chosen": -0.9500937461853027,
      "logits/rejected": -1.0103486776351929,
      "logps/chosen": -1.0463281869888306,
      "logps/rejected": -1.4567468166351318,
      "loss": 1.2082,
      "odds_ratio_loss": 0.5394900441169739,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05231640487909317,
      "rewards/margins": 0.020520929247140884,
      "rewards/rejected": -0.07283733785152435,
      "sft_loss": 1.0463281869888306,
      "step": 305
    },
    {
      "epoch": 0.992,
      "grad_norm": 4.944690721660475,
      "learning_rate": 4.231120786734689e-06,
      "logits/chosen": -1.0554213523864746,
      "logits/rejected": -1.2123026847839355,
      "logps/chosen": -1.0677235126495361,
      "logps/rejected": -1.427178978919983,
      "loss": 1.1548,
      "odds_ratio_loss": 0.5765484571456909,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05338617414236069,
      "rewards/margins": 0.01797277294099331,
      "rewards/rejected": -0.07135894149541855,
      "sft_loss": 1.0677235126495361,
      "step": 310
    },
    {
      "epoch": 1.008,
      "grad_norm": 3.9331589062338344,
      "learning_rate": 4.197173096220983e-06,
      "logits/chosen": -1.0465214252471924,
      "logits/rejected": -1.0950441360473633,
      "logps/chosen": -1.0325384140014648,
      "logps/rejected": -1.4658787250518799,
      "loss": 1.1007,
      "odds_ratio_loss": 0.5475068688392639,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.05162692070007324,
      "rewards/margins": 0.02166702225804329,
      "rewards/rejected": -0.07329393923282623,
      "sft_loss": 1.0325384140014648,
      "step": 315
    },
    {
      "epoch": 1.024,
      "grad_norm": 2.810636096476599,
      "learning_rate": 4.162634757195418e-06,
      "logits/chosen": -0.9247652888298035,
      "logits/rejected": -1.031118392944336,
      "logps/chosen": -0.9556388854980469,
      "logps/rejected": -1.2451565265655518,
      "loss": 0.9386,
      "odds_ratio_loss": 0.594650149345398,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.047781944274902344,
      "rewards/margins": 0.014475872740149498,
      "rewards/rejected": -0.06225781887769699,
      "sft_loss": 0.9556388854980469,
      "step": 320
    },
    {
      "epoch": 1.04,
      "grad_norm": 3.0448051628925454,
      "learning_rate": 4.127517789656772e-06,
      "logits/chosen": -0.9287153482437134,
      "logits/rejected": -1.0283979177474976,
      "logps/chosen": -1.0202507972717285,
      "logps/rejected": -1.2847683429718018,
      "loss": 1.0353,
      "odds_ratio_loss": 0.5959427952766418,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.051012538373470306,
      "rewards/margins": 0.01322587113827467,
      "rewards/rejected": -0.06423841416835785,
      "sft_loss": 1.0202507972717285,
      "step": 325
    },
    {
      "epoch": 1.056,
      "grad_norm": 4.1771500190151825,
      "learning_rate": 4.091834414977556e-06,
      "logits/chosen": -1.0129518508911133,
      "logits/rejected": -1.271144986152649,
      "logps/chosen": -1.018913984298706,
      "logps/rejected": -1.554570198059082,
      "loss": 0.9511,
      "odds_ratio_loss": 0.47632694244384766,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0509456992149353,
      "rewards/margins": 0.026782814413309097,
      "rewards/rejected": -0.0777285099029541,
      "sft_loss": 1.018913984298706,
      "step": 330
    },
    {
      "epoch": 1.072,
      "grad_norm": 6.50448197903886,
      "learning_rate": 4.055597051650731e-06,
      "logits/chosen": -1.0193812847137451,
      "logits/rejected": -1.1772067546844482,
      "logps/chosen": -0.9281299710273743,
      "logps/rejected": -1.3907569646835327,
      "loss": 0.9662,
      "odds_ratio_loss": 0.5411213040351868,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.046406496316194534,
      "rewards/margins": 0.023131350055336952,
      "rewards/rejected": -0.06953784823417664,
      "sft_loss": 0.9281299710273743,
      "step": 335
    },
    {
      "epoch": 1.088,
      "grad_norm": 5.600060917092442,
      "learning_rate": 4.018818310967843e-06,
      "logits/chosen": -1.1713335514068604,
      "logits/rejected": -1.2236130237579346,
      "logps/chosen": -1.0068949460983276,
      "logps/rejected": -1.1553858518600464,
      "loss": 0.9817,
      "odds_ratio_loss": 0.7148981094360352,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.05034474655985832,
      "rewards/margins": 0.007424544543027878,
      "rewards/rejected": -0.0577692911028862,
      "sft_loss": 1.0068949460983276,
      "step": 340
    },
    {
      "epoch": 1.104,
      "grad_norm": 5.092643762749835,
      "learning_rate": 3.981510992630055e-06,
      "logits/chosen": -0.9599639177322388,
      "logits/rejected": -1.189029574394226,
      "logps/chosen": -0.9695978164672852,
      "logps/rejected": -1.4548832178115845,
      "loss": 0.9953,
      "odds_ratio_loss": 0.5450643301010132,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.048479896038770676,
      "rewards/margins": 0.024264268577098846,
      "rewards/rejected": -0.07274416834115982,
      "sft_loss": 0.9695978164672852,
      "step": 345
    },
    {
      "epoch": 1.12,
      "grad_norm": 4.234304914475125,
      "learning_rate": 3.943688080293607e-06,
      "logits/chosen": -0.9754691123962402,
      "logits/rejected": -1.1199839115142822,
      "logps/chosen": -0.894666314125061,
      "logps/rejected": -1.577906847000122,
      "loss": 0.9707,
      "odds_ratio_loss": 0.45880669355392456,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.044733308255672455,
      "rewards/margins": 0.03416203707456589,
      "rewards/rejected": -0.07889535278081894,
      "sft_loss": 0.894666314125061,
      "step": 350
    },
    {
      "epoch": 1.1360000000000001,
      "grad_norm": 4.426257939120987,
      "learning_rate": 3.905362737051252e-06,
      "logits/chosen": -1.0120149850845337,
      "logits/rejected": -1.1284050941467285,
      "logps/chosen": -0.8394268751144409,
      "logps/rejected": -1.3487886190414429,
      "loss": 0.9606,
      "odds_ratio_loss": 0.5031847357749939,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.041971348226070404,
      "rewards/margins": 0.02546808496117592,
      "rewards/rejected": -0.06743943691253662,
      "sft_loss": 0.8394268751144409,
      "step": 355
    },
    {
      "epoch": 1.152,
      "grad_norm": 6.846072733066629,
      "learning_rate": 3.866548300851254e-06,
      "logits/chosen": -1.0661356449127197,
      "logits/rejected": -1.1536469459533691,
      "logps/chosen": -0.9550708532333374,
      "logps/rejected": -1.400836706161499,
      "loss": 0.9989,
      "odds_ratio_loss": 0.5404989123344421,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04775355011224747,
      "rewards/margins": 0.02228829450905323,
      "rewards/rejected": -0.07004183530807495,
      "sft_loss": 0.9550708532333374,
      "step": 360
    },
    {
      "epoch": 1.168,
      "grad_norm": 4.238159230888541,
      "learning_rate": 3.827258279855527e-06,
      "logits/chosen": -1.1802518367767334,
      "logits/rejected": -1.206186056137085,
      "logps/chosen": -1.0155624151229858,
      "logps/rejected": -1.3900431394577026,
      "loss": 0.9456,
      "odds_ratio_loss": 0.6698687672615051,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.05077812820672989,
      "rewards/margins": 0.01872403547167778,
      "rewards/rejected": -0.06950215995311737,
      "sft_loss": 1.0155624151229858,
      "step": 365
    },
    {
      "epoch": 1.184,
      "grad_norm": 3.7353385077354337,
      "learning_rate": 3.787506347738538e-06,
      "logits/chosen": -0.9678732752799988,
      "logits/rejected": -1.1043659448623657,
      "logps/chosen": -0.9759182929992676,
      "logps/rejected": -1.3616999387741089,
      "loss": 0.9721,
      "odds_ratio_loss": 0.6270595788955688,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0487959161400795,
      "rewards/margins": 0.019289087504148483,
      "rewards/rejected": -0.06808499991893768,
      "sft_loss": 0.9759182929992676,
      "step": 370
    },
    {
      "epoch": 1.2,
      "grad_norm": 4.7540211314956204,
      "learning_rate": 3.747306338928609e-06,
      "logits/chosen": -1.049630045890808,
      "logits/rejected": -1.2382904291152954,
      "logps/chosen": -0.938238263130188,
      "logps/rejected": -1.2308955192565918,
      "loss": 1.0158,
      "odds_ratio_loss": 0.5973559021949768,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04691191390156746,
      "rewards/margins": 0.01463286578655243,
      "rewards/rejected": -0.06154477596282959,
      "sft_loss": 0.938238263130188,
      "step": 375
    },
    {
      "epoch": 1.216,
      "grad_norm": 5.565360284878935,
      "learning_rate": 3.706672243793271e-06,
      "logits/chosen": -0.9532748460769653,
      "logits/rejected": -1.03403902053833,
      "logps/chosen": -0.9681148529052734,
      "logps/rejected": -1.2792482376098633,
      "loss": 0.9464,
      "odds_ratio_loss": 0.5441717505455017,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04840574413537979,
      "rewards/margins": 0.015556666068732738,
      "rewards/rejected": -0.0639624148607254,
      "sft_loss": 0.9681148529052734,
      "step": 380
    },
    {
      "epoch": 1.232,
      "grad_norm": 3.447185504214593,
      "learning_rate": 3.665618203770352e-06,
      "logits/chosen": -1.0210788249969482,
      "logits/rejected": -1.1396093368530273,
      "logps/chosen": -0.8457318544387817,
      "logps/rejected": -1.2301608324050903,
      "loss": 0.9478,
      "odds_ratio_loss": 0.5166088342666626,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04228659346699715,
      "rewards/margins": 0.01922144927084446,
      "rewards/rejected": -0.06150804087519646,
      "sft_loss": 0.8457318544387817,
      "step": 385
    },
    {
      "epoch": 1.248,
      "grad_norm": 3.3338981917851047,
      "learning_rate": 3.6241585064464846e-06,
      "logits/chosen": -1.2911275625228882,
      "logits/rejected": -1.1232960224151611,
      "logps/chosen": -0.8720508813858032,
      "logps/rejected": -1.23488187789917,
      "loss": 0.9435,
      "odds_ratio_loss": 0.5066360831260681,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04360254481434822,
      "rewards/margins": 0.01814154163002968,
      "rewards/rejected": -0.0617440864443779,
      "sft_loss": 0.8720508813858032,
      "step": 390
    },
    {
      "epoch": 1.264,
      "grad_norm": 3.567835515660478,
      "learning_rate": 3.582307580584759e-06,
      "logits/chosen": -0.959577739238739,
      "logits/rejected": -1.2563647031784058,
      "logps/chosen": -0.8968993425369263,
      "logps/rejected": -1.2443525791168213,
      "loss": 0.9669,
      "odds_ratio_loss": 0.521682620048523,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04484497010707855,
      "rewards/margins": 0.01737266220152378,
      "rewards/rejected": -0.062217630445957184,
      "sft_loss": 0.8968993425369263,
      "step": 395
    },
    {
      "epoch": 1.28,
      "grad_norm": 3.9755484861044956,
      "learning_rate": 3.5400799911032357e-06,
      "logits/chosen": -1.054391860961914,
      "logits/rejected": -1.017757534980774,
      "logps/chosen": -0.8325561285018921,
      "logps/rejected": -1.2529734373092651,
      "loss": 0.9323,
      "odds_ratio_loss": 0.4909485876560211,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.041627805680036545,
      "rewards/margins": 0.021020859479904175,
      "rewards/rejected": -0.06264867633581161,
      "sft_loss": 0.8325561285018921,
      "step": 400
    },
    {
      "epoch": 1.296,
      "grad_norm": 3.4685607506272476,
      "learning_rate": 3.4974904340060756e-06,
      "logits/chosen": -1.004164218902588,
      "logits/rejected": -1.320536732673645,
      "logps/chosen": -1.0670472383499146,
      "logps/rejected": -1.2909588813781738,
      "loss": 0.9978,
      "odds_ratio_loss": 0.6580184102058411,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05335236340761185,
      "rewards/margins": 0.011195586062967777,
      "rewards/rejected": -0.06454795598983765,
      "sft_loss": 1.0670472383499146,
      "step": 405
    },
    {
      "epoch": 1.312,
      "grad_norm": 4.3019598153539516,
      "learning_rate": 3.4545537312690565e-06,
      "logits/chosen": -1.1336114406585693,
      "logits/rejected": -1.1372759342193604,
      "logps/chosen": -0.9065093994140625,
      "logps/rejected": -1.415405511856079,
      "loss": 0.9679,
      "odds_ratio_loss": 0.4388805031776428,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.045325469225645065,
      "rewards/margins": 0.02544480562210083,
      "rewards/rejected": -0.0707702785730362,
      "sft_loss": 0.9065093994140625,
      "step": 410
    },
    {
      "epoch": 1.328,
      "grad_norm": 3.490824264107295,
      "learning_rate": 3.4112848256812374e-06,
      "logits/chosen": -1.0414103269577026,
      "logits/rejected": -1.164736032485962,
      "logps/chosen": -0.8454300761222839,
      "logps/rejected": -1.311577558517456,
      "loss": 0.9274,
      "odds_ratio_loss": 0.503560483455658,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.04227150231599808,
      "rewards/margins": 0.02330738492310047,
      "rewards/rejected": -0.0655788853764534,
      "sft_loss": 0.8454300761222839,
      "step": 415
    },
    {
      "epoch": 1.3439999999999999,
      "grad_norm": 8.789242453958117,
      "learning_rate": 3.3676987756445894e-06,
      "logits/chosen": -0.935681164264679,
      "logits/rejected": -1.1432523727416992,
      "logps/chosen": -0.9042528867721558,
      "logps/rejected": -1.587890386581421,
      "loss": 0.9137,
      "odds_ratio_loss": 0.47100192308425903,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04521264508366585,
      "rewards/margins": 0.0341818742454052,
      "rewards/rejected": -0.07939452677965164,
      "sft_loss": 0.9042528867721558,
      "step": 420
    },
    {
      "epoch": 1.3599999999999999,
      "grad_norm": 4.930177739224116,
      "learning_rate": 3.323810749933381e-06,
      "logits/chosen": -1.0698130130767822,
      "logits/rejected": -1.226738691329956,
      "logps/chosen": -0.8717530369758606,
      "logps/rejected": -1.4678089618682861,
      "loss": 0.9124,
      "odds_ratio_loss": 0.4427013397216797,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.04358765110373497,
      "rewards/margins": 0.02980278991162777,
      "rewards/rejected": -0.07339043915271759,
      "sft_loss": 0.8717530369758606,
      "step": 425
    },
    {
      "epoch": 1.376,
      "grad_norm": 7.570978097521412,
      "learning_rate": 3.2796360224151587e-06,
      "logits/chosen": -1.2224234342575073,
      "logits/rejected": -1.3439141511917114,
      "logps/chosen": -0.9194822311401367,
      "logps/rejected": -1.3092845678329468,
      "loss": 0.9594,
      "odds_ratio_loss": 0.5272402763366699,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.045974113047122955,
      "rewards/margins": 0.019490113481879234,
      "rewards/rejected": -0.06546422094106674,
      "sft_loss": 0.9194822311401367,
      "step": 430
    },
    {
      "epoch": 1.392,
      "grad_norm": 4.982061148370284,
      "learning_rate": 3.235189966735148e-06,
      "logits/chosen": -1.2072560787200928,
      "logits/rejected": -1.1339049339294434,
      "logps/chosen": -0.805392861366272,
      "logps/rejected": -1.594913125038147,
      "loss": 0.9259,
      "odds_ratio_loss": 0.406252384185791,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0402696393430233,
      "rewards/margins": 0.03947601094841957,
      "rewards/rejected": -0.07974565029144287,
      "sft_loss": 0.805392861366272,
      "step": 435
    },
    {
      "epoch": 1.408,
      "grad_norm": 7.692151482525084,
      "learning_rate": 3.1904880509659397e-06,
      "logits/chosen": -1.0890783071517944,
      "logits/rejected": -1.177278757095337,
      "logps/chosen": -0.9858812093734741,
      "logps/rejected": -1.204777479171753,
      "loss": 0.9667,
      "odds_ratio_loss": 0.637945830821991,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.049294065684080124,
      "rewards/margins": 0.010944806039333344,
      "rewards/rejected": -0.060238875448703766,
      "sft_loss": 0.9858812093734741,
      "step": 440
    },
    {
      "epoch": 1.424,
      "grad_norm": 4.492542501706703,
      "learning_rate": 3.1455458322242943e-06,
      "logits/chosen": -1.004298448562622,
      "logits/rejected": -1.1840426921844482,
      "logps/chosen": -0.9921313524246216,
      "logps/rejected": -1.2852861881256104,
      "loss": 0.9938,
      "odds_ratio_loss": 0.6350023150444031,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0496065691113472,
      "rewards/margins": 0.014657738618552685,
      "rewards/rejected": -0.06426431238651276,
      "sft_loss": 0.9921313524246216,
      "step": 445
    },
    {
      "epoch": 1.44,
      "grad_norm": 4.069388708509856,
      "learning_rate": 3.100378951256981e-06,
      "logits/chosen": -1.2047039270401,
      "logits/rejected": -1.2526731491088867,
      "logps/chosen": -0.8970128297805786,
      "logps/rejected": -1.087720274925232,
      "loss": 0.9649,
      "odds_ratio_loss": 0.6119378805160522,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04485064372420311,
      "rewards/margins": 0.009535368531942368,
      "rewards/rejected": -0.05438600853085518,
      "sft_loss": 0.8970128297805786,
      "step": 450
    },
    {
      "epoch": 1.456,
      "grad_norm": 4.1796047584641745,
      "learning_rate": 3.055003126997495e-06,
      "logits/chosen": -1.100693941116333,
      "logits/rejected": -1.1093151569366455,
      "logps/chosen": -0.9937971234321594,
      "logps/rejected": -1.3282194137573242,
      "loss": 0.9642,
      "odds_ratio_loss": 0.5750898122787476,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04968985915184021,
      "rewards/margins": 0.01672111637890339,
      "rewards/rejected": -0.06641098111867905,
      "sft_loss": 0.9937971234321594,
      "step": 455
    },
    {
      "epoch": 1.472,
      "grad_norm": 4.0507990154789075,
      "learning_rate": 3.0094341510955697e-06,
      "logits/chosen": -1.1363385915756226,
      "logits/rejected": -1.1738272905349731,
      "logps/chosen": -0.877064049243927,
      "logps/rejected": -1.4007574319839478,
      "loss": 0.959,
      "odds_ratio_loss": 0.5275009870529175,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04385320469737053,
      "rewards/margins": 0.026184672489762306,
      "rewards/rejected": -0.07003787159919739,
      "sft_loss": 0.877064049243927,
      "step": 460
    },
    {
      "epoch": 1.488,
      "grad_norm": 5.088249905544024,
      "learning_rate": 2.963687882421383e-06,
      "logits/chosen": -1.096651554107666,
      "logits/rejected": -1.1972352266311646,
      "logps/chosen": -0.9147326350212097,
      "logps/rejected": -1.253187894821167,
      "loss": 0.9591,
      "odds_ratio_loss": 0.5869191288948059,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04573662951588631,
      "rewards/margins": 0.01692276820540428,
      "rewards/rejected": -0.06265939772129059,
      "sft_loss": 0.9147326350212097,
      "step": 465
    },
    {
      "epoch": 1.504,
      "grad_norm": 4.206721558657063,
      "learning_rate": 2.9177802415463714e-06,
      "logits/chosen": -0.9876214265823364,
      "logits/rejected": -1.073717713356018,
      "logps/chosen": -0.7661324143409729,
      "logps/rejected": -1.2461497783660889,
      "loss": 0.9619,
      "odds_ratio_loss": 0.45304179191589355,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.038306623697280884,
      "rewards/margins": 0.0240008607506752,
      "rewards/rejected": -0.062307484447956085,
      "sft_loss": 0.7661324143409729,
      "step": 470
    },
    {
      "epoch": 1.52,
      "grad_norm": 3.67250528125703,
      "learning_rate": 2.871727205202563e-06,
      "logits/chosen": -1.0482373237609863,
      "logits/rejected": -1.1120043992996216,
      "logps/chosen": -0.950273871421814,
      "logps/rejected": -1.2787425518035889,
      "loss": 0.9498,
      "odds_ratio_loss": 0.599994421005249,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.047513701021671295,
      "rewards/margins": 0.016423430293798447,
      "rewards/rejected": -0.06393712759017944,
      "sft_loss": 0.950273871421814,
      "step": 475
    },
    {
      "epoch": 1.536,
      "grad_norm": 4.735321963888987,
      "learning_rate": 2.825544800722376e-06,
      "logits/chosen": -1.1098222732543945,
      "logits/rejected": -1.0207080841064453,
      "logps/chosen": -0.9046823382377625,
      "logps/rejected": -1.3911302089691162,
      "loss": 0.9629,
      "odds_ratio_loss": 0.5132350921630859,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.04523410648107529,
      "rewards/margins": 0.0243223924189806,
      "rewards/rejected": -0.06955650448799133,
      "sft_loss": 0.9046823382377625,
      "step": 480
    },
    {
      "epoch": 1.552,
      "grad_norm": 3.159767853907868,
      "learning_rate": 2.7792491004607984e-06,
      "logits/chosen": -1.0096070766448975,
      "logits/rejected": -1.0770386457443237,
      "logps/chosen": -0.9277147054672241,
      "logps/rejected": -1.344773530960083,
      "loss": 0.9485,
      "odds_ratio_loss": 0.5044125318527222,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.046385735273361206,
      "rewards/margins": 0.020852940157055855,
      "rewards/rejected": -0.06723867356777191,
      "sft_loss": 0.9277147054672241,
      "step": 485
    },
    {
      "epoch": 1.568,
      "grad_norm": 4.013671913402214,
      "learning_rate": 2.732856216201906e-06,
      "logits/chosen": -1.063245415687561,
      "logits/rejected": -1.1233893632888794,
      "logps/chosen": -0.9281940460205078,
      "logps/rejected": -1.3444063663482666,
      "loss": 0.9529,
      "odds_ratio_loss": 0.489690363407135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04640969634056091,
      "rewards/margins": 0.020810618996620178,
      "rewards/rejected": -0.06722031533718109,
      "sft_loss": 0.9281940460205078,
      "step": 490
    },
    {
      "epoch": 1.584,
      "grad_norm": 4.31314553790671,
      "learning_rate": 2.6863822935516546e-06,
      "logits/chosen": -1.0234626531600952,
      "logits/rejected": -1.1978650093078613,
      "logps/chosen": -0.9632134437561035,
      "logps/rejected": -1.206189751625061,
      "loss": 0.9733,
      "odds_ratio_loss": 0.5884974002838135,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.048160675913095474,
      "rewards/margins": 0.012148816138505936,
      "rewards/rejected": -0.06030949205160141,
      "sft_loss": 0.9632134437561035,
      "step": 495
    },
    {
      "epoch": 1.6,
      "grad_norm": 3.3125411606876445,
      "learning_rate": 2.639843506318899e-06,
      "logits/chosen": -0.9256556630134583,
      "logits/rejected": -1.1392202377319336,
      "logps/chosen": -0.8793741464614868,
      "logps/rejected": -1.3441039323806763,
      "loss": 0.9562,
      "odds_ratio_loss": 0.510412335395813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04396871104836464,
      "rewards/margins": 0.023236487060785294,
      "rewards/rejected": -0.06720519065856934,
      "sft_loss": 0.8793741464614868,
      "step": 500
    },
    {
      "epoch": 1.616,
      "grad_norm": 5.866811891887456,
      "learning_rate": 2.593256050886603e-06,
      "logits/chosen": -0.8647163510322571,
      "logits/rejected": -1.0798470973968506,
      "logps/chosen": -0.9747751951217651,
      "logps/rejected": -1.2537850141525269,
      "loss": 0.9531,
      "odds_ratio_loss": 0.5470961332321167,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0487387590110302,
      "rewards/margins": 0.01395049411803484,
      "rewards/rejected": -0.06268925964832306,
      "sft_loss": 0.9747751951217651,
      "step": 505
    },
    {
      "epoch": 1.6320000000000001,
      "grad_norm": 4.074172203592654,
      "learning_rate": 2.5466361405751914e-06,
      "logits/chosen": -0.9186518788337708,
      "logits/rejected": -1.1423577070236206,
      "logps/chosen": -0.9840022921562195,
      "logps/rejected": -1.3202232122421265,
      "loss": 0.949,
      "odds_ratio_loss": 0.564594566822052,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.049200110137462616,
      "rewards/margins": 0.01681104302406311,
      "rewards/rejected": -0.06601114571094513,
      "sft_loss": 0.9840022921562195,
      "step": 510
    },
    {
      "epoch": 1.6480000000000001,
      "grad_norm": 5.002204843627367,
      "learning_rate": 2.5e-06,
      "logits/chosen": -0.8605943918228149,
      "logits/rejected": -0.9710782766342163,
      "logps/chosen": -0.8691279292106628,
      "logps/rejected": -1.2410802841186523,
      "loss": 0.9795,
      "odds_ratio_loss": 0.5254562497138977,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04345639795064926,
      "rewards/margins": 0.018597617745399475,
      "rewards/rejected": -0.06205401569604874,
      "sft_loss": 0.8691279292106628,
      "step": 515
    },
    {
      "epoch": 1.6640000000000001,
      "grad_norm": 8.858448372423531,
      "learning_rate": 2.4533638594248094e-06,
      "logits/chosen": -0.9867407083511353,
      "logits/rejected": -1.0888783931732178,
      "logps/chosen": -0.9163697957992554,
      "logps/rejected": -1.1703190803527832,
      "loss": 0.9221,
      "odds_ratio_loss": 0.5501306056976318,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.045818496495485306,
      "rewards/margins": 0.012697461061179638,
      "rewards/rejected": -0.05851595476269722,
      "sft_loss": 0.9163697957992554,
      "step": 520
    },
    {
      "epoch": 1.6800000000000002,
      "grad_norm": 4.445755031880296,
      "learning_rate": 2.406743949113397e-06,
      "logits/chosen": -0.9282479286193848,
      "logits/rejected": -1.1179710626602173,
      "logps/chosen": -0.872515082359314,
      "logps/rejected": -1.3933911323547363,
      "loss": 0.9773,
      "odds_ratio_loss": 0.5091356039047241,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04362575337290764,
      "rewards/margins": 0.026043808087706566,
      "rewards/rejected": -0.06966955959796906,
      "sft_loss": 0.872515082359314,
      "step": 525
    },
    {
      "epoch": 1.696,
      "grad_norm": 3.6413598914526486,
      "learning_rate": 2.360156493681102e-06,
      "logits/chosen": -0.910436749458313,
      "logits/rejected": -1.0247265100479126,
      "logps/chosen": -0.9496439695358276,
      "logps/rejected": -1.3968040943145752,
      "loss": 0.9491,
      "odds_ratio_loss": 0.48868894577026367,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.0474822036921978,
      "rewards/margins": 0.0223580040037632,
      "rewards/rejected": -0.069840207695961,
      "sft_loss": 0.9496439695358276,
      "step": 530
    },
    {
      "epoch": 1.712,
      "grad_norm": 4.294695857238959,
      "learning_rate": 2.3136177064483462e-06,
      "logits/chosen": -1.0878788232803345,
      "logits/rejected": -1.283992052078247,
      "logps/chosen": -1.003148078918457,
      "logps/rejected": -1.4421515464782715,
      "loss": 0.9586,
      "odds_ratio_loss": 0.49956098198890686,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05015740543603897,
      "rewards/margins": 0.021950174123048782,
      "rewards/rejected": -0.07210757583379745,
      "sft_loss": 1.003148078918457,
      "step": 535
    },
    {
      "epoch": 1.728,
      "grad_norm": 4.183928338953188,
      "learning_rate": 2.2671437837980943e-06,
      "logits/chosen": -1.0223979949951172,
      "logits/rejected": -0.9681426286697388,
      "logps/chosen": -0.8269376754760742,
      "logps/rejected": -1.141745924949646,
      "loss": 0.8999,
      "odds_ratio_loss": 0.5333009958267212,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04134688898921013,
      "rewards/margins": 0.0157404113560915,
      "rewards/rejected": -0.05708730220794678,
      "sft_loss": 0.8269376754760742,
      "step": 540
    },
    {
      "epoch": 1.744,
      "grad_norm": 4.65432905249252,
      "learning_rate": 2.2207508995392024e-06,
      "logits/chosen": -0.9679327011108398,
      "logits/rejected": -1.0990087985992432,
      "logps/chosen": -0.9268666505813599,
      "logps/rejected": -1.2932831048965454,
      "loss": 0.9613,
      "odds_ratio_loss": 0.5248704552650452,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04634333401918411,
      "rewards/margins": 0.018320811912417412,
      "rewards/rejected": -0.06466414779424667,
      "sft_loss": 0.9268666505813599,
      "step": 545
    },
    {
      "epoch": 1.76,
      "grad_norm": 3.9586982529561205,
      "learning_rate": 2.1744551992776247e-06,
      "logits/chosen": -1.0059576034545898,
      "logits/rejected": -1.0268685817718506,
      "logps/chosen": -0.9964353442192078,
      "logps/rejected": -1.6557565927505493,
      "loss": 0.9815,
      "odds_ratio_loss": 0.49777206778526306,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.04982176423072815,
      "rewards/margins": 0.032966069877147675,
      "rewards/rejected": -0.08278782665729523,
      "sft_loss": 0.9964353442192078,
      "step": 550
    },
    {
      "epoch": 1.776,
      "grad_norm": 9.562472852188419,
      "learning_rate": 2.1282727947974373e-06,
      "logits/chosen": -0.9853806495666504,
      "logits/rejected": -1.1351251602172852,
      "logps/chosen": -0.8706343770027161,
      "logps/rejected": -1.4279866218566895,
      "loss": 0.9807,
      "odds_ratio_loss": 0.4323902130126953,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.043531715869903564,
      "rewards/margins": 0.02786761149764061,
      "rewards/rejected": -0.07139933109283447,
      "sft_loss": 0.8706343770027161,
      "step": 555
    },
    {
      "epoch": 1.792,
      "grad_norm": 4.3083896592948845,
      "learning_rate": 2.082219758453629e-06,
      "logits/chosen": -0.8532622456550598,
      "logits/rejected": -1.0556373596191406,
      "logps/chosen": -0.9265901446342468,
      "logps/rejected": -1.6611661911010742,
      "loss": 0.9808,
      "odds_ratio_loss": 0.47050419449806213,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04632951319217682,
      "rewards/margins": 0.03672880306839943,
      "rewards/rejected": -0.08305831253528595,
      "sft_loss": 0.9265901446342468,
      "step": 560
    },
    {
      "epoch": 1.808,
      "grad_norm": 4.988596811372773,
      "learning_rate": 2.036312117578617e-06,
      "logits/chosen": -0.8543407320976257,
      "logits/rejected": -1.035388708114624,
      "logps/chosen": -0.9570035934448242,
      "logps/rejected": -1.4836640357971191,
      "loss": 0.9573,
      "odds_ratio_loss": 0.4832231104373932,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04785018041729927,
      "rewards/margins": 0.02633301541209221,
      "rewards/rejected": -0.07418319582939148,
      "sft_loss": 0.9570035934448242,
      "step": 565
    },
    {
      "epoch": 1.8239999999999998,
      "grad_norm": 3.4762921171914463,
      "learning_rate": 1.990565848904431e-06,
      "logits/chosen": -0.8208405375480652,
      "logits/rejected": -0.940590500831604,
      "logps/chosen": -0.8431119918823242,
      "logps/rejected": -1.3455002307891846,
      "loss": 0.9561,
      "odds_ratio_loss": 0.4739263653755188,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04215560108423233,
      "rewards/margins": 0.025119412690401077,
      "rewards/rejected": -0.06727501004934311,
      "sft_loss": 0.8431119918823242,
      "step": 570
    },
    {
      "epoch": 1.8399999999999999,
      "grad_norm": 4.955237689773966,
      "learning_rate": 1.9449968730025055e-06,
      "logits/chosen": -0.9368654489517212,
      "logits/rejected": -1.018002986907959,
      "logps/chosen": -1.0393065214157104,
      "logps/rejected": -1.3489172458648682,
      "loss": 0.9946,
      "odds_ratio_loss": 0.5891560316085815,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.05196532607078552,
      "rewards/margins": 0.01548053603619337,
      "rewards/rejected": -0.06744585931301117,
      "sft_loss": 1.0393065214157104,
      "step": 575
    },
    {
      "epoch": 1.8559999999999999,
      "grad_norm": 4.122518133723736,
      "learning_rate": 1.899621048743019e-06,
      "logits/chosen": -0.8472858667373657,
      "logits/rejected": -0.9123756289482117,
      "logps/chosen": -0.8696669340133667,
      "logps/rejected": -1.3730722665786743,
      "loss": 0.9332,
      "odds_ratio_loss": 0.504239559173584,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.043483346700668335,
      "rewards/margins": 0.025170262902975082,
      "rewards/rejected": -0.06865362077951431,
      "sft_loss": 0.8696669340133667,
      "step": 580
    },
    {
      "epoch": 1.8719999999999999,
      "grad_norm": 5.090413419223807,
      "learning_rate": 1.854454167775706e-06,
      "logits/chosen": -0.8803795576095581,
      "logits/rejected": -1.011498212814331,
      "logps/chosen": -0.8195604085922241,
      "logps/rejected": -1.5066629648208618,
      "loss": 0.9053,
      "odds_ratio_loss": 0.4221072793006897,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.040978025645017624,
      "rewards/margins": 0.03435512259602547,
      "rewards/rejected": -0.07533314824104309,
      "sft_loss": 0.8195604085922241,
      "step": 585
    },
    {
      "epoch": 1.888,
      "grad_norm": 3.962979618406243,
      "learning_rate": 1.8095119490340618e-06,
      "logits/chosen": -0.9934213757514954,
      "logits/rejected": -1.028438925743103,
      "logps/chosen": -0.9060677289962769,
      "logps/rejected": -1.4857866764068604,
      "loss": 0.9753,
      "odds_ratio_loss": 0.6394127011299133,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04530338943004608,
      "rewards/margins": 0.028985943645238876,
      "rewards/rejected": -0.07428933680057526,
      "sft_loss": 0.9060677289962769,
      "step": 590
    },
    {
      "epoch": 1.904,
      "grad_norm": 4.384891130460857,
      "learning_rate": 1.764810033264852e-06,
      "logits/chosen": -0.907455563545227,
      "logits/rejected": -1.0589368343353271,
      "logps/chosen": -1.0206191539764404,
      "logps/rejected": -1.4559767246246338,
      "loss": 0.9771,
      "odds_ratio_loss": 0.5931872129440308,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0510309636592865,
      "rewards/margins": 0.02176786959171295,
      "rewards/rejected": -0.07279883325099945,
      "sft_loss": 1.0206191539764404,
      "step": 595
    },
    {
      "epoch": 1.92,
      "grad_norm": 3.4134689622305383,
      "learning_rate": 1.7203639775848423e-06,
      "logits/chosen": -1.0145207643508911,
      "logits/rejected": -0.9972650408744812,
      "logps/chosen": -0.956381618976593,
      "logps/rejected": -1.3823316097259521,
      "loss": 0.9631,
      "odds_ratio_loss": 0.5243911743164062,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04781908541917801,
      "rewards/margins": 0.021297505125403404,
      "rewards/rejected": -0.06911659240722656,
      "sft_loss": 0.956381618976593,
      "step": 600
    },
    {
      "epoch": 1.936,
      "grad_norm": 4.505148742615646,
      "learning_rate": 1.6761892500666195e-06,
      "logits/chosen": -1.023581624031067,
      "logits/rejected": -1.027942419052124,
      "logps/chosen": -0.933574378490448,
      "logps/rejected": -1.2963403463363647,
      "loss": 0.9786,
      "odds_ratio_loss": 0.5001336336135864,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.04667872190475464,
      "rewards/margins": 0.018138296902179718,
      "rewards/rejected": -0.06481701880693436,
      "sft_loss": 0.933574378490448,
      "step": 605
    },
    {
      "epoch": 1.952,
      "grad_norm": 4.83889942986056,
      "learning_rate": 1.632301224355411e-06,
      "logits/chosen": -0.8887473940849304,
      "logits/rejected": -0.9829289317131042,
      "logps/chosen": -1.0025461912155151,
      "logps/rejected": -1.2573713064193726,
      "loss": 0.9546,
      "odds_ratio_loss": 0.6103852987289429,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0501273088157177,
      "rewards/margins": 0.012741250917315483,
      "rewards/rejected": -0.06286855787038803,
      "sft_loss": 1.0025461912155151,
      "step": 610
    },
    {
      "epoch": 1.968,
      "grad_norm": 4.517273923883564,
      "learning_rate": 1.5887151743187634e-06,
      "logits/chosen": -0.9073827862739563,
      "logits/rejected": -1.0887978076934814,
      "logps/chosen": -0.9604595899581909,
      "logps/rejected": -1.4289729595184326,
      "loss": 0.9811,
      "odds_ratio_loss": 0.545965850353241,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.048022981733083725,
      "rewards/margins": 0.023425664752721786,
      "rewards/rejected": -0.07144864648580551,
      "sft_loss": 0.9604595899581909,
      "step": 615
    },
    {
      "epoch": 1.984,
      "grad_norm": 3.1757477627169908,
      "learning_rate": 1.5454462687309445e-06,
      "logits/chosen": -1.015500783920288,
      "logits/rejected": -1.0785764455795288,
      "logps/chosen": -0.9297005534172058,
      "logps/rejected": -1.5386453866958618,
      "loss": 0.9563,
      "odds_ratio_loss": 0.542974054813385,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04648502543568611,
      "rewards/margins": 0.03044723905622959,
      "rewards/rejected": -0.07693226635456085,
      "sft_loss": 0.9297005534172058,
      "step": 620
    },
    {
      "epoch": 2.0,
      "grad_norm": 4.242978905390128,
      "learning_rate": 1.502509565993925e-06,
      "logits/chosen": -0.9173771142959595,
      "logits/rejected": -1.0122209787368774,
      "logps/chosen": -0.9267832040786743,
      "logps/rejected": -1.5153340101242065,
      "loss": 0.8898,
      "odds_ratio_loss": 0.45579010248184204,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.04633915796875954,
      "rewards/margins": 0.02942754328250885,
      "rewards/rejected": -0.07576669752597809,
      "sft_loss": 0.9267832040786743,
      "step": 625
    },
    {
      "epoch": 2.016,
      "grad_norm": 3.553798975041404,
      "learning_rate": 1.4599200088967652e-06,
      "logits/chosen": -0.8619723320007324,
      "logits/rejected": -1.0356481075286865,
      "logps/chosen": -0.8099665641784668,
      "logps/rejected": -1.3428175449371338,
      "loss": 0.8054,
      "odds_ratio_loss": 0.42676934599876404,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.04049833118915558,
      "rewards/margins": 0.02664254978299141,
      "rewards/rejected": -0.06714087724685669,
      "sft_loss": 0.8099665641784668,
      "step": 630
    },
    {
      "epoch": 2.032,
      "grad_norm": 4.569902596370669,
      "learning_rate": 1.4176924194152405e-06,
      "logits/chosen": -0.9947516322135925,
      "logits/rejected": -1.0760139226913452,
      "logps/chosen": -0.822666347026825,
      "logps/rejected": -1.4580928087234497,
      "loss": 0.8194,
      "odds_ratio_loss": 0.408845990896225,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.041133321821689606,
      "rewards/margins": 0.03177132084965706,
      "rewards/rejected": -0.07290463894605637,
      "sft_loss": 0.822666347026825,
      "step": 635
    },
    {
      "epoch": 2.048,
      "grad_norm": 3.58850101082042,
      "learning_rate": 1.3758414935535147e-06,
      "logits/chosen": -0.8080762624740601,
      "logits/rejected": -0.8731328845024109,
      "logps/chosen": -0.6912453174591064,
      "logps/rejected": -1.2947412729263306,
      "loss": 0.7991,
      "odds_ratio_loss": 0.39434918761253357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03456226736307144,
      "rewards/margins": 0.030174797400832176,
      "rewards/rejected": -0.06473706662654877,
      "sft_loss": 0.6912453174591064,
      "step": 640
    },
    {
      "epoch": 2.064,
      "grad_norm": 3.1095303302424204,
      "learning_rate": 1.3343817962296485e-06,
      "logits/chosen": -0.9426826238632202,
      "logits/rejected": -0.9611290097236633,
      "logps/chosen": -0.7455964088439941,
      "logps/rejected": -1.3733656406402588,
      "loss": 0.8219,
      "odds_ratio_loss": 0.400754451751709,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.037279821932315826,
      "rewards/margins": 0.03138846904039383,
      "rewards/rejected": -0.06866829097270966,
      "sft_loss": 0.7455964088439941,
      "step": 645
    },
    {
      "epoch": 2.08,
      "grad_norm": 3.045256483819498,
      "learning_rate": 1.293327756206729e-06,
      "logits/chosen": -0.8983833193778992,
      "logits/rejected": -0.9661597013473511,
      "logps/chosen": -0.6603522300720215,
      "logps/rejected": -1.5160930156707764,
      "loss": 0.7579,
      "odds_ratio_loss": 0.3125734031200409,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.033017612993717194,
      "rewards/margins": 0.042787034064531326,
      "rewards/rejected": -0.07580464333295822,
      "sft_loss": 0.6603522300720215,
      "step": 650
    },
    {
      "epoch": 2.096,
      "grad_norm": 3.1717779800652592,
      "learning_rate": 1.252693661071391e-06,
      "logits/chosen": -1.157251238822937,
      "logits/rejected": -1.0623748302459717,
      "logps/chosen": -0.7086284160614014,
      "logps/rejected": -1.514593243598938,
      "loss": 0.7865,
      "odds_ratio_loss": 0.33622443675994873,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.035431426018476486,
      "rewards/margins": 0.04029824212193489,
      "rewards/rejected": -0.07572966068983078,
      "sft_loss": 0.7086284160614014,
      "step": 655
    },
    {
      "epoch": 2.112,
      "grad_norm": 2.7205852811983093,
      "learning_rate": 1.2124936522614622e-06,
      "logits/chosen": -1.0289185047149658,
      "logits/rejected": -1.098934292793274,
      "logps/chosen": -0.7381170988082886,
      "logps/rejected": -1.4106049537658691,
      "loss": 0.7903,
      "odds_ratio_loss": 0.45428943634033203,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.03690585121512413,
      "rewards/margins": 0.033624399453401566,
      "rewards/rejected": -0.0705302506685257,
      "sft_loss": 0.7381170988082886,
      "step": 660
    },
    {
      "epoch": 2.128,
      "grad_norm": 2.8528967143088666,
      "learning_rate": 1.1727417201444735e-06,
      "logits/chosen": -1.0716520547866821,
      "logits/rejected": -1.0720608234405518,
      "logps/chosen": -0.7356234788894653,
      "logps/rejected": -1.2771320343017578,
      "loss": 0.797,
      "odds_ratio_loss": 0.41821280121803284,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.036781176924705505,
      "rewards/margins": 0.027075421065092087,
      "rewards/rejected": -0.0638565942645073,
      "sft_loss": 0.7356234788894653,
      "step": 665
    },
    {
      "epoch": 2.144,
      "grad_norm": 3.7706399394026615,
      "learning_rate": 1.1334516991487473e-06,
      "logits/chosen": -0.7660879492759705,
      "logits/rejected": -1.0078188180923462,
      "logps/chosen": -0.7483173608779907,
      "logps/rejected": -1.2434841394424438,
      "loss": 0.7863,
      "odds_ratio_loss": 0.3927895426750183,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.037415869534015656,
      "rewards/margins": 0.02475833147764206,
      "rewards/rejected": -0.062174201011657715,
      "sft_loss": 0.7483173608779907,
      "step": 670
    },
    {
      "epoch": 2.16,
      "grad_norm": 3.5703943571070482,
      "learning_rate": 1.094637262948749e-06,
      "logits/chosen": -0.9008601307868958,
      "logits/rejected": -1.0412832498550415,
      "logps/chosen": -0.821381688117981,
      "logps/rejected": -1.3886059522628784,
      "loss": 0.7675,
      "odds_ratio_loss": 0.4320302903652191,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04106908664107323,
      "rewards/margins": 0.02836121991276741,
      "rewards/rejected": -0.06943030655384064,
      "sft_loss": 0.821381688117981,
      "step": 675
    },
    {
      "epoch": 2.176,
      "grad_norm": 2.9837572151294065,
      "learning_rate": 1.0563119197063934e-06,
      "logits/chosen": -1.058257818222046,
      "logits/rejected": -1.0895675420761108,
      "logps/chosen": -0.914572536945343,
      "logps/rejected": -1.3927881717681885,
      "loss": 0.7969,
      "odds_ratio_loss": 0.46075814962387085,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.04572862759232521,
      "rewards/margins": 0.023910781368613243,
      "rewards/rejected": -0.0696394145488739,
      "sft_loss": 0.914572536945343,
      "step": 680
    },
    {
      "epoch": 2.192,
      "grad_norm": 3.2497337072742414,
      "learning_rate": 1.018489007369945e-06,
      "logits/chosen": -0.8853160738945007,
      "logits/rejected": -0.8999375104904175,
      "logps/chosen": -0.7777100801467896,
      "logps/rejected": -1.448999047279358,
      "loss": 0.804,
      "odds_ratio_loss": 0.39251843094825745,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03888550028204918,
      "rewards/margins": 0.03356444090604782,
      "rewards/rejected": -0.0724499449133873,
      "sft_loss": 0.7777100801467896,
      "step": 685
    },
    {
      "epoch": 2.208,
      "grad_norm": 3.029373367507165,
      "learning_rate": 9.81181689032158e-07,
      "logits/chosen": -0.8918782472610474,
      "logits/rejected": -0.8753920793533325,
      "logps/chosen": -0.6909304857254028,
      "logps/rejected": -1.2901936769485474,
      "loss": 0.7751,
      "odds_ratio_loss": 0.4143063426017761,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.03454653173685074,
      "rewards/margins": 0.029963159933686256,
      "rewards/rejected": -0.06450968980789185,
      "sft_loss": 0.6909304857254028,
      "step": 690
    },
    {
      "epoch": 2.224,
      "grad_norm": 4.5746404005970405,
      "learning_rate": 9.444029483492703e-07,
      "logits/chosen": -1.0063061714172363,
      "logits/rejected": -1.0239533185958862,
      "logps/chosen": -0.6867491006851196,
      "logps/rejected": -1.245091199874878,
      "loss": 0.7732,
      "odds_ratio_loss": 0.4617982804775238,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.03433745354413986,
      "rewards/margins": 0.027917101979255676,
      "rewards/rejected": -0.06225455552339554,
      "sft_loss": 0.6867491006851196,
      "step": 695
    },
    {
      "epoch": 2.24,
      "grad_norm": 3.694255891644154,
      "learning_rate": 9.081655850224449e-07,
      "logits/chosen": -1.0207760334014893,
      "logits/rejected": -1.1230969429016113,
      "logps/chosen": -0.8350968360900879,
      "logps/rejected": -1.5500547885894775,
      "loss": 0.7583,
      "odds_ratio_loss": 0.42910847067832947,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.041754838079214096,
      "rewards/margins": 0.03574790060520172,
      "rewards/rejected": -0.07750274240970612,
      "sft_loss": 0.8350968360900879,
      "step": 700
    },
    {
      "epoch": 2.2560000000000002,
      "grad_norm": 3.994487822092531,
      "learning_rate": 8.724822103432282e-07,
      "logits/chosen": -0.8827289342880249,
      "logits/rejected": -0.9654695391654968,
      "logps/chosen": -0.7088466286659241,
      "logps/rejected": -1.281082034111023,
      "loss": 0.7482,
      "odds_ratio_loss": 0.3990407884120941,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.035442329943180084,
      "rewards/margins": 0.028611773625016212,
      "rewards/rejected": -0.06405410915613174,
      "sft_loss": 0.7088466286659241,
      "step": 705
    },
    {
      "epoch": 2.2720000000000002,
      "grad_norm": 3.3050191320581166,
      "learning_rate": 8.373652428045831e-07,
      "logits/chosen": -1.0878264904022217,
      "logits/rejected": -1.0405784845352173,
      "logps/chosen": -0.65968257188797,
      "logps/rejected": -1.3744075298309326,
      "loss": 0.7722,
      "odds_ratio_loss": 0.3181590139865875,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.03298413008451462,
      "rewards/margins": 0.03573625162243843,
      "rewards/rejected": -0.06872038543224335,
      "sft_loss": 0.65968257188797,
      "step": 710
    },
    {
      "epoch": 2.288,
      "grad_norm": 3.8117555833594943,
      "learning_rate": 8.028269037790171e-07,
      "logits/chosen": -0.8991669416427612,
      "logits/rejected": -0.9795140027999878,
      "logps/chosen": -0.8547693490982056,
      "logps/rejected": -1.5010745525360107,
      "loss": 0.7688,
      "odds_ratio_loss": 0.42176875472068787,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.04273846372961998,
      "rewards/margins": 0.03231526166200638,
      "rewards/rejected": -0.07505372911691666,
      "sft_loss": 0.8547693490982056,
      "step": 715
    },
    {
      "epoch": 2.304,
      "grad_norm": 6.060881007391729,
      "learning_rate": 7.688792132653111e-07,
      "logits/chosen": -0.8518077731132507,
      "logits/rejected": -0.9652704000473022,
      "logps/chosen": -0.7316969633102417,
      "logps/rejected": -1.5251853466033936,
      "loss": 0.7621,
      "odds_ratio_loss": 0.3367399573326111,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.03658485412597656,
      "rewards/margins": 0.03967442363500595,
      "rewards/rejected": -0.07625927031040192,
      "sft_loss": 0.7316969633102417,
      "step": 720
    },
    {
      "epoch": 2.32,
      "grad_norm": 2.9281573250934363,
      "learning_rate": 7.355339857053318e-07,
      "logits/chosen": -0.8477774858474731,
      "logits/rejected": -1.018747329711914,
      "logps/chosen": -0.7693859934806824,
      "logps/rejected": -1.3998419046401978,
      "loss": 0.7714,
      "odds_ratio_loss": 0.3711521625518799,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03846929594874382,
      "rewards/margins": 0.031522803008556366,
      "rewards/rejected": -0.06999210268259048,
      "sft_loss": 0.7693859934806824,
      "step": 725
    },
    {
      "epoch": 2.336,
      "grad_norm": 3.3575685063670213,
      "learning_rate": 7.028028258723818e-07,
      "logits/chosen": -0.9588859677314758,
      "logits/rejected": -1.0327568054199219,
      "logps/chosen": -0.7077163457870483,
      "logps/rejected": -1.3171709775924683,
      "loss": 0.7932,
      "odds_ratio_loss": 0.3643357455730438,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.03538581356406212,
      "rewards/margins": 0.030472736805677414,
      "rewards/rejected": -0.06585855782032013,
      "sft_loss": 0.7077163457870483,
      "step": 730
    },
    {
      "epoch": 2.352,
      "grad_norm": 3.07955633424655,
      "learning_rate": 6.706971248325151e-07,
      "logits/chosen": -0.8883659243583679,
      "logits/rejected": -1.0441168546676636,
      "logps/chosen": -0.8074674606323242,
      "logps/rejected": -1.4699620008468628,
      "loss": 0.776,
      "odds_ratio_loss": 0.39879804849624634,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.04037337377667427,
      "rewards/margins": 0.033124733716249466,
      "rewards/rejected": -0.07349809259176254,
      "sft_loss": 0.8074674606323242,
      "step": 735
    },
    {
      "epoch": 2.368,
      "grad_norm": 3.6213614434043593,
      "learning_rate": 6.392280559802341e-07,
      "logits/chosen": -0.9162646532058716,
      "logits/rejected": -0.9279802441596985,
      "logps/chosen": -0.7218812704086304,
      "logps/rejected": -1.4897241592407227,
      "loss": 0.7637,
      "odds_ratio_loss": 0.411416620016098,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0360940620303154,
      "rewards/margins": 0.03839214891195297,
      "rewards/rejected": -0.07448621094226837,
      "sft_loss": 0.7218812704086304,
      "step": 740
    },
    {
      "epoch": 2.384,
      "grad_norm": 4.1017905530653245,
      "learning_rate": 6.084065711499326e-07,
      "logits/chosen": -0.84368896484375,
      "logits/rejected": -0.9179951548576355,
      "logps/chosen": -0.6008902192115784,
      "logps/rejected": -1.4554692506790161,
      "loss": 0.7279,
      "odds_ratio_loss": 0.3157138526439667,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.030044514685869217,
      "rewards/margins": 0.04272894933819771,
      "rewards/rejected": -0.07277346402406693,
      "sft_loss": 0.6008902192115784,
      "step": 745
    },
    {
      "epoch": 2.4,
      "grad_norm": 3.8487814271047758,
      "learning_rate": 5.782433968044495e-07,
      "logits/chosen": -0.8399477005004883,
      "logits/rejected": -0.9413140416145325,
      "logps/chosen": -0.8139437437057495,
      "logps/rejected": -1.5214980840682983,
      "loss": 0.7662,
      "odds_ratio_loss": 0.5000792741775513,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.040697190910577774,
      "rewards/margins": 0.03537772223353386,
      "rewards/rejected": -0.07607491314411163,
      "sft_loss": 0.8139437437057495,
      "step": 750
    },
    {
      "epoch": 2.416,
      "grad_norm": 3.305488495552487,
      "learning_rate": 5.487490303020576e-07,
      "logits/chosen": -0.9991554021835327,
      "logits/rejected": -0.944214940071106,
      "logps/chosen": -0.7551851272583008,
      "logps/rejected": -1.2736141681671143,
      "loss": 0.7806,
      "odds_ratio_loss": 0.4188924729824066,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.03775925561785698,
      "rewards/margins": 0.025921454653143883,
      "rewards/rejected": -0.06368071585893631,
      "sft_loss": 0.7551851272583008,
      "step": 755
    },
    {
      "epoch": 2.432,
      "grad_norm": 3.6380522040854566,
      "learning_rate": 5.199337362431792e-07,
      "logits/chosen": -0.8715981245040894,
      "logits/rejected": -1.0095371007919312,
      "logps/chosen": -0.716741681098938,
      "logps/rejected": -1.7061433792114258,
      "loss": 0.7998,
      "odds_ratio_loss": 0.40209442377090454,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0358370803296566,
      "rewards/margins": 0.04947008565068245,
      "rewards/rejected": -0.08530716598033905,
      "sft_loss": 0.716741681098938,
      "step": 760
    },
    {
      "epoch": 2.448,
      "grad_norm": 4.799248860716653,
      "learning_rate": 4.918075428981148e-07,
      "logits/chosen": -0.9698484539985657,
      "logits/rejected": -1.0252515077590942,
      "logps/chosen": -0.7276674509048462,
      "logps/rejected": -1.2364145517349243,
      "loss": 0.7519,
      "odds_ratio_loss": 0.4512792229652405,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.03638336807489395,
      "rewards/margins": 0.025437360629439354,
      "rewards/rejected": -0.06182073429226875,
      "sft_loss": 0.7276674509048462,
      "step": 765
    },
    {
      "epoch": 2.464,
      "grad_norm": 3.975888652914653,
      "learning_rate": 4.643802387170118e-07,
      "logits/chosen": -0.784801185131073,
      "logits/rejected": -0.8751401901245117,
      "logps/chosen": -0.736864447593689,
      "logps/rejected": -1.4113752841949463,
      "loss": 0.7689,
      "odds_ratio_loss": 0.41703367233276367,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03684321790933609,
      "rewards/margins": 0.0337255522608757,
      "rewards/rejected": -0.07056877762079239,
      "sft_loss": 0.736864447593689,
      "step": 770
    },
    {
      "epoch": 2.48,
      "grad_norm": 3.7017186080610873,
      "learning_rate": 4.376613689232978e-07,
      "logits/chosen": -0.9608728289604187,
      "logits/rejected": -1.000018835067749,
      "logps/chosen": -0.8199416399002075,
      "logps/rejected": -1.469847321510315,
      "loss": 0.7588,
      "odds_ratio_loss": 0.43195098638534546,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.04099708050489426,
      "rewards/margins": 0.03249528631567955,
      "rewards/rejected": -0.0734923705458641,
      "sft_loss": 0.8199416399002075,
      "step": 775
    },
    {
      "epoch": 2.496,
      "grad_norm": 3.1981116471330764,
      "learning_rate": 4.1166023219176176e-07,
      "logits/chosen": -0.8377124667167664,
      "logits/rejected": -1.0116181373596191,
      "logps/chosen": -0.7824350595474243,
      "logps/rejected": -1.6547685861587524,
      "loss": 0.7523,
      "odds_ratio_loss": 0.32834020256996155,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.039121754467487335,
      "rewards/margins": 0.043616678565740585,
      "rewards/rejected": -0.08273844420909882,
      "sft_loss": 0.7824350595474243,
      "step": 780
    },
    {
      "epoch": 2.512,
      "grad_norm": 3.7082914287151945,
      "learning_rate": 3.863858774124385e-07,
      "logits/chosen": -1.0026713609695435,
      "logits/rejected": -1.0367608070373535,
      "logps/chosen": -0.6959080696105957,
      "logps/rejected": -1.4358288049697876,
      "loss": 0.7563,
      "odds_ratio_loss": 0.3291171193122864,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.03479539602994919,
      "rewards/margins": 0.036996036767959595,
      "rewards/rejected": -0.07179144024848938,
      "sft_loss": 0.6959080696105957,
      "step": 785
    },
    {
      "epoch": 2.528,
      "grad_norm": 2.9819183882511004,
      "learning_rate": 3.618471005414215e-07,
      "logits/chosen": -1.1593316793441772,
      "logits/rejected": -1.1186132431030273,
      "logps/chosen": -0.5521804094314575,
      "logps/rejected": -1.3308180570602417,
      "loss": 0.7382,
      "odds_ratio_loss": 0.40159231424331665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.027609020471572876,
      "rewards/margins": 0.03893188014626503,
      "rewards/rejected": -0.0665409117937088,
      "sft_loss": 0.5521804094314575,
      "step": 790
    },
    {
      "epoch": 2.544,
      "grad_norm": 3.3567110818810755,
      "learning_rate": 3.380524415397049e-07,
      "logits/chosen": -0.8402732014656067,
      "logits/rejected": -1.0374484062194824,
      "logps/chosen": -0.803017795085907,
      "logps/rejected": -1.2986352443695068,
      "loss": 0.7742,
      "odds_ratio_loss": 0.46145597100257874,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04015089198946953,
      "rewards/margins": 0.0247808750718832,
      "rewards/rejected": -0.06493176519870758,
      "sft_loss": 0.803017795085907,
      "step": 795
    },
    {
      "epoch": 2.56,
      "grad_norm": 3.3420577148113924,
      "learning_rate": 3.150101814011136e-07,
      "logits/chosen": -1.031750202178955,
      "logits/rejected": -1.02663254737854,
      "logps/chosen": -0.7611913681030273,
      "logps/rejected": -1.1853117942810059,
      "loss": 0.7785,
      "odds_ratio_loss": 0.5039734244346619,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.03805956989526749,
      "rewards/margins": 0.021206015720963478,
      "rewards/rejected": -0.05926559120416641,
      "sft_loss": 0.7611913681030273,
      "step": 800
    },
    {
      "epoch": 2.576,
      "grad_norm": 3.246898163699306,
      "learning_rate": 2.927283392703606e-07,
      "logits/chosen": -0.8511770367622375,
      "logits/rejected": -0.9470258951187134,
      "logps/chosen": -0.7670743465423584,
      "logps/rejected": -1.1585172414779663,
      "loss": 0.7977,
      "odds_ratio_loss": 0.4781486988067627,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03835371881723404,
      "rewards/margins": 0.019572149962186813,
      "rewards/rejected": -0.057925861328840256,
      "sft_loss": 0.7670743465423584,
      "step": 805
    },
    {
      "epoch": 2.592,
      "grad_norm": 3.9992460548702438,
      "learning_rate": 2.712146696522305e-07,
      "logits/chosen": -1.022859811782837,
      "logits/rejected": -1.129640817642212,
      "logps/chosen": -0.716964840888977,
      "logps/rejected": -1.246201992034912,
      "loss": 0.7552,
      "odds_ratio_loss": 0.4771757125854492,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.03584824129939079,
      "rewards/margins": 0.026461860164999962,
      "rewards/rejected": -0.062310099601745605,
      "sft_loss": 0.716964840888977,
      "step": 810
    },
    {
      "epoch": 2.608,
      "grad_norm": 3.5945402118846403,
      "learning_rate": 2.504766597128666e-07,
      "logits/chosen": -0.9580585360527039,
      "logits/rejected": -0.9554399251937866,
      "logps/chosen": -0.8108547925949097,
      "logps/rejected": -1.315863847732544,
      "loss": 0.7575,
      "odds_ratio_loss": 0.44800853729248047,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.040542736649513245,
      "rewards/margins": 0.025250453501939774,
      "rewards/rejected": -0.06579320132732391,
      "sft_loss": 0.8108547925949097,
      "step": 815
    },
    {
      "epoch": 2.624,
      "grad_norm": 3.288445101598415,
      "learning_rate": 2.3052152667409289e-07,
      "logits/chosen": -0.9488247036933899,
      "logits/rejected": -0.8991857767105103,
      "logps/chosen": -0.7179723978042603,
      "logps/rejected": -1.3861563205718994,
      "loss": 0.7752,
      "odds_ratio_loss": 0.3779214322566986,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03589861840009689,
      "rewards/margins": 0.03340919688344002,
      "rewards/rejected": -0.06930781900882721,
      "sft_loss": 0.7179723978042603,
      "step": 820
    },
    {
      "epoch": 2.64,
      "grad_norm": 4.333903778456222,
      "learning_rate": 2.1135621530168488e-07,
      "logits/chosen": -1.028704285621643,
      "logits/rejected": -0.9637089967727661,
      "logps/chosen": -0.6309131383895874,
      "logps/rejected": -1.2894703149795532,
      "loss": 0.7935,
      "odds_ratio_loss": 0.3709859549999237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03154565393924713,
      "rewards/margins": 0.03292786702513695,
      "rewards/rejected": -0.06447352468967438,
      "sft_loss": 0.6309131383895874,
      "step": 825
    },
    {
      "epoch": 2.656,
      "grad_norm": 3.1671393797255414,
      "learning_rate": 1.9298739548845813e-07,
      "logits/chosen": -0.9645661115646362,
      "logits/rejected": -0.9816401600837708,
      "logps/chosen": -0.8160637021064758,
      "logps/rejected": -1.374236822128296,
      "loss": 0.8091,
      "odds_ratio_loss": 0.4034562110900879,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.04080318659543991,
      "rewards/margins": 0.027908671647310257,
      "rewards/rejected": -0.06871185451745987,
      "sft_loss": 0.8160637021064758,
      "step": 830
    },
    {
      "epoch": 2.672,
      "grad_norm": 3.093675577889326,
      "learning_rate": 1.7542145993301896e-07,
      "logits/chosen": -0.9770193099975586,
      "logits/rejected": -1.0430421829223633,
      "logps/chosen": -0.6873968839645386,
      "logps/rejected": -1.183712124824524,
      "loss": 0.7636,
      "odds_ratio_loss": 0.42723220586776733,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.03436984866857529,
      "rewards/margins": 0.02481575682759285,
      "rewards/rejected": -0.05918560549616814,
      "sft_loss": 0.6873968839645386,
      "step": 835
    },
    {
      "epoch": 2.6879999999999997,
      "grad_norm": 3.044985818298562,
      "learning_rate": 1.5866452191498488e-07,
      "logits/chosen": -0.8661419153213501,
      "logits/rejected": -1.0225311517715454,
      "logps/chosen": -0.7636631727218628,
      "logps/rejected": -1.324355959892273,
      "loss": 0.7742,
      "odds_ratio_loss": 0.3950451910495758,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.03818316012620926,
      "rewards/margins": 0.028034633025527,
      "rewards/rejected": -0.06621779501438141,
      "sft_loss": 0.7636631727218628,
      "step": 840
    },
    {
      "epoch": 2.7039999999999997,
      "grad_norm": 3.49139680447691,
      "learning_rate": 1.4272241316744456e-07,
      "logits/chosen": -0.9768983125686646,
      "logits/rejected": -1.0676887035369873,
      "logps/chosen": -0.8691689372062683,
      "logps/rejected": -1.5023279190063477,
      "loss": 0.7386,
      "odds_ratio_loss": 0.36283236742019653,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.043458450585603714,
      "rewards/margins": 0.03165794909000397,
      "rewards/rejected": -0.07511639595031738,
      "sft_loss": 0.8691689372062683,
      "step": 845
    },
    {
      "epoch": 2.7199999999999998,
      "grad_norm": 3.4065842852697674,
      "learning_rate": 1.2760068184740597e-07,
      "logits/chosen": -0.9277766942977905,
      "logits/rejected": -0.9539043307304382,
      "logps/chosen": -0.6972528100013733,
      "logps/rejected": -1.4623371362686157,
      "loss": 0.711,
      "odds_ratio_loss": 0.32903754711151123,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.03486264497041702,
      "rewards/margins": 0.03825421258807182,
      "rewards/rejected": -0.07311685383319855,
      "sft_loss": 0.6972528100013733,
      "step": 850
    },
    {
      "epoch": 2.7359999999999998,
      "grad_norm": 4.258492525068464,
      "learning_rate": 1.133045906049321e-07,
      "logits/chosen": -0.9535223245620728,
      "logits/rejected": -0.967192530632019,
      "logps/chosen": -0.7654585242271423,
      "logps/rejected": -1.2852718830108643,
      "loss": 0.7165,
      "odds_ratio_loss": 0.4885888695716858,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.038272932171821594,
      "rewards/margins": 0.025990664958953857,
      "rewards/rejected": -0.06426359713077545,
      "sft_loss": 0.7654585242271423,
      "step": 855
    },
    {
      "epoch": 2.752,
      "grad_norm": 3.0541533378277403,
      "learning_rate": 9.983911475163727e-08,
      "logits/chosen": -0.9215459823608398,
      "logits/rejected": -1.025499939918518,
      "logps/chosen": -0.6536693572998047,
      "logps/rejected": -1.464961051940918,
      "loss": 0.7491,
      "odds_ratio_loss": 0.32264453172683716,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.032683465629816055,
      "rewards/margins": 0.04056458920240402,
      "rewards/rejected": -0.07324805110692978,
      "sft_loss": 0.6536693572998047,
      "step": 860
    },
    {
      "epoch": 2.768,
      "grad_norm": 3.5262579401831413,
      "learning_rate": 8.720894052918566e-08,
      "logits/chosen": -0.9398951530456543,
      "logits/rejected": -1.0038102865219116,
      "logps/chosen": -0.6747088432312012,
      "logps/rejected": -1.5684466361999512,
      "loss": 0.7404,
      "odds_ratio_loss": 0.35855698585510254,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.03373543918132782,
      "rewards/margins": 0.044686902314424515,
      "rewards/rejected": -0.07842233031988144,
      "sft_loss": 0.6747088432312012,
      "step": 865
    },
    {
      "epoch": 2.784,
      "grad_norm": 4.66998911832725,
      "learning_rate": 7.541846347838915e-08,
      "logits/chosen": -0.9948604702949524,
      "logits/rejected": -0.9804432988166809,
      "logps/chosen": -0.7318006753921509,
      "logps/rejected": -1.360406517982483,
      "loss": 0.7496,
      "odds_ratio_loss": 0.4134978652000427,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.036590032279491425,
      "rewards/margins": 0.03143029659986496,
      "rewards/rejected": -0.06802032887935638,
      "sft_loss": 0.7318006753921509,
      "step": 870
    },
    {
      "epoch": 2.8,
      "grad_norm": 3.657269423224737,
      "learning_rate": 6.447178690947492e-08,
      "logits/chosen": -0.8408433794975281,
      "logits/rejected": -1.0504786968231201,
      "logps/chosen": -0.6820616126060486,
      "logps/rejected": -1.6608028411865234,
      "loss": 0.7865,
      "odds_ratio_loss": 0.2992148697376251,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.03410308063030243,
      "rewards/margins": 0.04893706366419792,
      "rewards/rejected": -0.08304014801979065,
      "sft_loss": 0.6820616126060486,
      "step": 875
    },
    {
      "epoch": 2.816,
      "grad_norm": 3.377129562618715,
      "learning_rate": 5.437272047405712e-08,
      "logits/chosen": -0.8684722781181335,
      "logits/rejected": -0.871597945690155,
      "logps/chosen": -0.7219066619873047,
      "logps/rejected": -1.350375771522522,
      "loss": 0.7529,
      "odds_ratio_loss": 0.3972117006778717,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.036095328629016876,
      "rewards/margins": 0.031423456966876984,
      "rewards/rejected": -0.06751878559589386,
      "sft_loss": 0.7219066619873047,
      "step": 880
    },
    {
      "epoch": 2.832,
      "grad_norm": 3.1663560374696633,
      "learning_rate": 4.512477883930527e-08,
      "logits/chosen": -0.9551021456718445,
      "logits/rejected": -1.0277870893478394,
      "logps/chosen": -0.7899004817008972,
      "logps/rejected": -1.5054200887680054,
      "loss": 0.8151,
      "odds_ratio_loss": 0.44721174240112305,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03949502483010292,
      "rewards/margins": 0.03577598184347153,
      "rewards/rejected": -0.07527101784944534,
      "sft_loss": 0.7899004817008972,
      "step": 885
    },
    {
      "epoch": 2.848,
      "grad_norm": 4.553277973933315,
      "learning_rate": 3.673118046477159e-08,
      "logits/chosen": -0.9583977460861206,
      "logits/rejected": -0.9128350019454956,
      "logps/chosen": -0.7262309789657593,
      "logps/rejected": -1.2458374500274658,
      "loss": 0.7505,
      "odds_ratio_loss": 0.4251146912574768,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.0363115519285202,
      "rewards/margins": 0.025980323553085327,
      "rewards/rejected": -0.06229187175631523,
      "sft_loss": 0.7262309789657593,
      "step": 890
    },
    {
      "epoch": 2.864,
      "grad_norm": 3.38523058441021,
      "learning_rate": 2.9194846482308026e-08,
      "logits/chosen": -0.9198935627937317,
      "logits/rejected": -1.0665795803070068,
      "logps/chosen": -0.7491655349731445,
      "logps/rejected": -1.439427137374878,
      "loss": 0.7622,
      "odds_ratio_loss": 0.3993758261203766,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.037458278238773346,
      "rewards/margins": 0.03451308235526085,
      "rewards/rejected": -0.0719713643193245,
      "sft_loss": 0.7491655349731445,
      "step": 895
    },
    {
      "epoch": 2.88,
      "grad_norm": 5.053660983515992,
      "learning_rate": 2.251839967945535e-08,
      "logits/chosen": -0.9375904202461243,
      "logits/rejected": -1.060706377029419,
      "logps/chosen": -0.8767011761665344,
      "logps/rejected": -1.344328761100769,
      "loss": 0.766,
      "odds_ratio_loss": 0.5356343984603882,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.04383506253361702,
      "rewards/margins": 0.02338137850165367,
      "rewards/rejected": -0.06721644103527069,
      "sft_loss": 0.8767011761665344,
      "step": 900
    },
    {
      "epoch": 2.896,
      "grad_norm": 4.052840811239487,
      "learning_rate": 1.6704163586663825e-08,
      "logits/chosen": -0.9922626614570618,
      "logits/rejected": -0.952767014503479,
      "logps/chosen": -0.7710371017456055,
      "logps/rejected": -1.4237831830978394,
      "loss": 0.8119,
      "odds_ratio_loss": 0.37999871373176575,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.03855185955762863,
      "rewards/margins": 0.032637301832437515,
      "rewards/rejected": -0.07118916511535645,
      "sft_loss": 0.7710371017456055,
      "step": 905
    },
    {
      "epoch": 2.912,
      "grad_norm": 3.5836749421401146,
      "learning_rate": 1.1754161668660612e-08,
      "logits/chosen": -0.8529298901557922,
      "logits/rejected": -0.905608057975769,
      "logps/chosen": -0.7730112671852112,
      "logps/rejected": -1.4924733638763428,
      "loss": 0.7818,
      "odds_ratio_loss": 0.40126991271972656,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03865056484937668,
      "rewards/margins": 0.035973113030195236,
      "rewards/rejected": -0.07462368160486221,
      "sft_loss": 0.7730112671852112,
      "step": 910
    },
    {
      "epoch": 2.928,
      "grad_norm": 4.257912492562235,
      "learning_rate": 7.670116620245304e-09,
      "logits/chosen": -0.9177722930908203,
      "logits/rejected": -1.0528053045272827,
      "logps/chosen": -0.7789991497993469,
      "logps/rejected": -1.4093081951141357,
      "loss": 0.7862,
      "odds_ratio_loss": 0.41386303305625916,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.03894995525479317,
      "rewards/margins": 0.0315154567360878,
      "rewards/rejected": -0.07046540826559067,
      "sft_loss": 0.7789991497993469,
      "step": 915
    },
    {
      "epoch": 2.944,
      "grad_norm": 4.5782153540538575,
      "learning_rate": 4.453449766758933e-09,
      "logits/chosen": -1.0013253688812256,
      "logits/rejected": -0.9696464538574219,
      "logps/chosen": -0.7406843900680542,
      "logps/rejected": -1.4851348400115967,
      "loss": 0.7435,
      "odds_ratio_loss": 0.3701634407043457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.037034209817647934,
      "rewards/margins": 0.03722252696752548,
      "rewards/rejected": -0.07425674051046371,
      "sft_loss": 0.7406843900680542,
      "step": 920
    },
    {
      "epoch": 2.96,
      "grad_norm": 3.8405956220288906,
      "learning_rate": 2.1052805694365964e-09,
      "logits/chosen": -0.9112046360969543,
      "logits/rejected": -0.953746497631073,
      "logps/chosen": -0.9097617864608765,
      "logps/rejected": -1.2347737550735474,
      "loss": 0.787,
      "odds_ratio_loss": 0.5556334853172302,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04548809304833412,
      "rewards/margins": 0.016250593587756157,
      "rewards/rejected": -0.061738692224025726,
      "sft_loss": 0.9097617864608765,
      "step": 925
    },
    {
      "epoch": 2.976,
      "grad_norm": 3.3132646219123676,
      "learning_rate": 6.264262358129936e-10,
      "logits/chosen": -0.8271144032478333,
      "logits/rejected": -1.076704502105713,
      "logps/chosen": -0.8304189443588257,
      "logps/rejected": -1.3862361907958984,
      "loss": 0.7539,
      "odds_ratio_loss": 0.4186479449272156,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.04152094945311546,
      "rewards/margins": 0.027790868654847145,
      "rewards/rejected": -0.06931181252002716,
      "sft_loss": 0.8304189443588257,
      "step": 930
    },
    {
      "epoch": 2.992,
      "grad_norm": 3.846388656199648,
      "learning_rate": 1.7401435318531444e-11,
      "logits/chosen": -1.0561763048171997,
      "logits/rejected": -1.0017073154449463,
      "logps/chosen": -0.6843768358230591,
      "logps/rejected": -1.4440100193023682,
      "loss": 0.7977,
      "odds_ratio_loss": 0.4135671555995941,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.034218840301036835,
      "rewards/margins": 0.03798165172338486,
      "rewards/rejected": -0.07220049947500229,
      "sft_loss": 0.6843768358230591,
      "step": 935
    },
    {
      "epoch": 2.9952,
      "step": 936,
      "total_flos": 172904605286400.0,
      "train_loss": 0.9994224820636276,
      "train_runtime": 29817.6595,
      "train_samples_per_second": 2.012,
      "train_steps_per_second": 0.031
    }
  ],
  "logging_steps": 5,
  "max_steps": 936,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100.0,
  "total_flos": 172904605286400.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}