{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 1875,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 2.6595744680851065e-08,
      "logits/chosen": 0.1680368036031723,
      "logits/rejected": 0.19713005423545837,
      "logps/chosen": -393.7256164550781,
      "logps/rejected": -273.0241394042969,
      "loss": 0.4136,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.6595744680851066e-07,
      "logits/chosen": 0.1247146800160408,
      "logits/rejected": 0.257645845413208,
      "logps/chosen": -338.6689147949219,
      "logps/rejected": -323.99322509765625,
      "loss": 0.3312,
      "rewards/accuracies": 0.4722222089767456,
      "rewards/chosen": 0.00012141283514210954,
      "rewards/margins": 0.00012025787145830691,
      "rewards/rejected": 1.154946744463814e-06,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.319148936170213e-07,
      "logits/chosen": 0.11776206642389297,
      "logits/rejected": 0.2888662815093994,
      "logps/chosen": -342.19476318359375,
      "logps/rejected": -288.05242919921875,
      "loss": 0.3206,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -3.678434222820215e-05,
      "rewards/margins": -0.0001559042138978839,
      "rewards/rejected": 0.00011911986803170294,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.97872340425532e-07,
      "logits/chosen": 0.05618368461728096,
      "logits/rejected": 0.21764644980430603,
      "logps/chosen": -348.53204345703125,
      "logps/rejected": -298.0450134277344,
      "loss": 0.3618,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -7.375622953986749e-05,
      "rewards/margins": -8.439932571491227e-05,
      "rewards/rejected": 1.0643097994034179e-05,
      "step": 30
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0638297872340427e-06,
      "logits/chosen": 0.10220017284154892,
      "logits/rejected": 0.21337243914604187,
      "logps/chosen": -362.06536865234375,
      "logps/rejected": -303.7508544921875,
      "loss": 0.323,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 4.589342097460758e-06,
      "rewards/margins": 6.563085662492085e-06,
      "rewards/rejected": -1.973749021999538e-06,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3297872340425533e-06,
      "logits/chosen": 0.17307938635349274,
      "logits/rejected": 0.2492096871137619,
      "logps/chosen": -357.7861633300781,
      "logps/rejected": -262.5830993652344,
      "loss": 0.3332,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.00020471049356274307,
      "rewards/margins": 0.0001833896094467491,
      "rewards/rejected": 2.1320887753972784e-05,
      "step": 50
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.595744680851064e-06,
      "logits/chosen": 0.09367106109857559,
      "logits/rejected": 0.27534765005111694,
      "logps/chosen": -365.8818359375,
      "logps/rejected": -301.4277038574219,
      "loss": 0.325,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00033300495124422014,
      "rewards/margins": 0.00035451535950414836,
      "rewards/rejected": -2.1510411897907034e-05,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8617021276595745e-06,
      "logits/chosen": 0.0627938061952591,
      "logits/rejected": 0.26493480801582336,
      "logps/chosen": -373.78619384765625,
      "logps/rejected": -256.60858154296875,
      "loss": 0.3137,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.0005881573888473213,
      "rewards/margins": 0.0005571565707214177,
      "rewards/rejected": 3.1000781746115535e-05,
      "step": 70
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.1276595744680853e-06,
      "logits/chosen": 0.07975228130817413,
      "logits/rejected": 0.18873457610607147,
      "logps/chosen": -351.62811279296875,
      "logps/rejected": -268.86175537109375,
      "loss": 0.3427,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.0011552410433068871,
      "rewards/margins": 0.0011365157552063465,
      "rewards/rejected": 1.8725168047240004e-05,
      "step": 80
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.393617021276596e-06,
      "logits/chosen": 0.10162068903446198,
      "logits/rejected": 0.2277887761592865,
      "logps/chosen": -334.11505126953125,
      "logps/rejected": -277.01123046875,
      "loss": 0.3284,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.0022371101658791304,
      "rewards/margins": 0.0022209202870726585,
      "rewards/rejected": 1.619003523956053e-05,
      "step": 90
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6595744680851065e-06,
      "logits/chosen": 0.09252412617206573,
      "logits/rejected": 0.27979540824890137,
      "logps/chosen": -377.9844970703125,
      "logps/rejected": -289.419677734375,
      "loss": 0.3082,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.00396860484033823,
      "rewards/margins": 0.002819900633767247,
      "rewards/rejected": 0.001148704090155661,
      "step": 100
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.9255319148936174e-06,
      "logits/chosen": 0.10446289926767349,
      "logits/rejected": 0.19878003001213074,
      "logps/chosen": -369.4442443847656,
      "logps/rejected": -287.75946044921875,
      "loss": 0.33,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.006103604566305876,
      "rewards/margins": 0.0044761234894394875,
      "rewards/rejected": 0.0016274808440357447,
      "step": 110
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.191489361702128e-06,
      "logits/chosen": 0.1083141341805458,
      "logits/rejected": 0.2530149817466736,
      "logps/chosen": -323.8044128417969,
      "logps/rejected": -300.437255859375,
      "loss": 0.3517,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008737698197364807,
      "rewards/margins": 0.006680602673441172,
      "rewards/rejected": 0.0020570955239236355,
      "step": 120
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.457446808510639e-06,
      "logits/chosen": 0.1410514861345291,
      "logits/rejected": 0.2637427747249603,
      "logps/chosen": -328.4124450683594,
      "logps/rejected": -277.685791015625,
      "loss": 0.3177,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.01055722963064909,
      "rewards/margins": 0.010405505076050758,
      "rewards/rejected": 0.0001517242781119421,
      "step": 130
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.723404255319149e-06,
      "logits/chosen": 0.15804480016231537,
      "logits/rejected": 0.24627968668937683,
      "logps/chosen": -309.4755554199219,
      "logps/rejected": -280.66851806640625,
      "loss": 0.3346,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.011000417172908783,
      "rewards/margins": 0.013859736733138561,
      "rewards/rejected": -0.0028593200258910656,
      "step": 140
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.98936170212766e-06,
      "logits/chosen": 0.17287364602088928,
      "logits/rejected": 0.2511886954307556,
      "logps/chosen": -343.8251953125,
      "logps/rejected": -319.0948181152344,
      "loss": 0.3016,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.01205248199403286,
      "rewards/margins": 0.023393789306282997,
      "rewards/rejected": -0.011341306380927563,
      "step": 150
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.255319148936171e-06,
      "logits/chosen": 0.13911333680152893,
      "logits/rejected": 0.3385851979255676,
      "logps/chosen": -357.8835144042969,
      "logps/rejected": -313.824951171875,
      "loss": 0.3054,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0063235037960112095,
      "rewards/margins": 0.03163896128535271,
      "rewards/rejected": -0.03796246275305748,
      "step": 160
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.521276595744681e-06,
      "logits/chosen": 0.20543082058429718,
      "logits/rejected": 0.286451518535614,
      "logps/chosen": -427.0508728027344,
      "logps/rejected": -397.6676025390625,
      "loss": 0.2817,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.032296378165483475,
      "rewards/margins": 0.06459534913301468,
      "rewards/rejected": -0.09689172357320786,
      "step": 170
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.787234042553192e-06,
      "logits/chosen": 0.17126373946666718,
      "logits/rejected": 0.25217366218566895,
      "logps/chosen": -439.67840576171875,
      "logps/rejected": -456.2896423339844,
      "loss": 0.292,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.10424342006444931,
      "rewards/margins": 0.08777059614658356,
      "rewards/rejected": -0.19201400876045227,
      "step": 180
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999982660399688e-06,
      "logits/chosen": 0.16082295775413513,
      "logits/rejected": 0.2546341121196747,
      "logps/chosen": -461.6935119628906,
      "logps/rejected": -500.5730895996094,
      "loss": 0.2571,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12413378804922104,
      "rewards/margins": 0.08718255907297134,
      "rewards/rejected": -0.21131631731987,
      "step": 190
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.99937579964398e-06,
      "logits/chosen": 0.10711757093667984,
      "logits/rejected": 0.21617627143859863,
      "logps/chosen": -522.9395141601562,
      "logps/rejected": -570.61083984375,
      "loss": 0.2479,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.15697070956230164,
      "rewards/margins": 0.11644785106182098,
      "rewards/rejected": -0.2734185755252838,
      "step": 200
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.9979021993870645e-06,
      "logits/chosen": 0.22454562783241272,
      "logits/rejected": 0.3751831650733948,
      "logps/chosen": -618.0147094726562,
      "logps/rejected": -661.6094970703125,
      "loss": 0.2742,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2251421958208084,
      "rewards/margins": 0.1425340622663498,
      "rewards/rejected": -0.3676762282848358,
      "step": 210
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995562370647553e-06,
      "logits/chosen": 0.2090134620666504,
      "logits/rejected": 0.2619253993034363,
      "logps/chosen": -585.8196411132812,
      "logps/rejected": -695.9527587890625,
      "loss": 0.2409,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.23935195803642273,
      "rewards/margins": 0.16129860281944275,
      "rewards/rejected": -0.4006505608558655,
      "step": 220
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.992357124836838e-06,
      "logits/chosen": 0.21562126278877258,
      "logits/rejected": 0.30403202772140503,
      "logps/chosen": -583.1571044921875,
      "logps/rejected": -664.2890625,
      "loss": 0.2766,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.22509849071502686,
      "rewards/margins": 0.1467365324497223,
      "rewards/rejected": -0.37183502316474915,
      "step": 230
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9882875734777044e-06,
      "logits/chosen": 0.09697722643613815,
      "logits/rejected": 0.240617036819458,
      "logps/chosen": -562.3460083007812,
      "logps/rejected": -621.7235107421875,
      "loss": 0.2826,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.21679548919200897,
      "rewards/margins": 0.14000090956687927,
      "rewards/rejected": -0.35679641366004944,
      "step": 240
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.983355127818882e-06,
      "logits/chosen": 0.13605690002441406,
      "logits/rejected": 0.2448308765888214,
      "logps/chosen": -535.7677001953125,
      "logps/rejected": -593.754638671875,
      "loss": 0.2796,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.18055127561092377,
      "rewards/margins": 0.10934337228536606,
      "rewards/rejected": -0.28989464044570923,
      "step": 250
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.977561498345639e-06,
      "logits/chosen": 0.10516633838415146,
      "logits/rejected": 0.23380067944526672,
      "logps/chosen": -482.5274353027344,
      "logps/rejected": -600.8789672851562,
      "loss": 0.2452,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1688566356897354,
      "rewards/margins": 0.15834231674671173,
      "rewards/rejected": -0.32719895243644714,
      "step": 260
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.970908694186624e-06,
      "logits/chosen": 0.1699642688035965,
      "logits/rejected": 0.2693479657173157,
      "logps/chosen": -505.5724182128906,
      "logps/rejected": -634.6489868164062,
      "loss": 0.2573,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.1955190747976303,
      "rewards/margins": 0.16697654128074646,
      "rewards/rejected": -0.3624956011772156,
      "step": 270
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9633990224171305e-06,
      "logits/chosen": 0.16180387139320374,
      "logits/rejected": 0.2788015604019165,
      "logps/chosen": -564.1033935546875,
      "logps/rejected": -646.7479248046875,
      "loss": 0.2496,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.2246371954679489,
      "rewards/margins": 0.13526742160320282,
      "rewards/rejected": -0.35990458726882935,
      "step": 280
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.955035087259046e-06,
      "logits/chosen": 0.1516331136226654,
      "logits/rejected": 0.20982889831066132,
      "logps/chosen": -544.4299926757812,
      "logps/rejected": -642.7681274414062,
      "loss": 0.2689,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.22307808697223663,
      "rewards/margins": 0.14021030068397522,
      "rewards/rejected": -0.36328837275505066,
      "step": 290
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.945819789177756e-06,
      "logits/chosen": 0.0853557214140892,
      "logits/rejected": 0.24294111132621765,
      "logps/chosen": -587.2979125976562,
      "logps/rejected": -675.4234619140625,
      "loss": 0.2403,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.22484929859638214,
      "rewards/margins": 0.1554943472146988,
      "rewards/rejected": -0.38034364581108093,
      "step": 300
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.935756323876306e-06,
      "logits/chosen": 0.1488361358642578,
      "logits/rejected": 0.22367210686206818,
      "logps/chosen": -545.3068237304688,
      "logps/rejected": -659.3970336914062,
      "loss": 0.2389,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.18365520238876343,
      "rewards/margins": 0.17183148860931396,
      "rewards/rejected": -0.3554866909980774,
      "step": 310
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.924848181187199e-06,
      "logits/chosen": 0.08838851749897003,
      "logits/rejected": 0.21153846383094788,
      "logps/chosen": -565.9591064453125,
      "logps/rejected": -670.2896728515625,
      "loss": 0.2343,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.22553689777851105,
      "rewards/margins": 0.15771272778511047,
      "rewards/rejected": -0.3832496404647827,
      "step": 320
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.913099143862173e-06,
      "logits/chosen": 0.11703666299581528,
      "logits/rejected": 0.19987936317920685,
      "logps/chosen": -552.8609008789062,
      "logps/rejected": -671.7772216796875,
      "loss": 0.2388,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.2358858585357666,
      "rewards/margins": 0.1468653529882431,
      "rewards/rejected": -0.3827512562274933,
      "step": 330
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.900513286260416e-06,
      "logits/chosen": 0.12850847840309143,
      "logits/rejected": 0.22665157914161682,
      "logps/chosen": -592.2838134765625,
      "logps/rejected": -709.990966796875,
      "loss": 0.2315,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.2650226652622223,
      "rewards/margins": 0.156343013048172,
      "rewards/rejected": -0.4213656783103943,
      "step": 340
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.887094972935645e-06,
      "logits/chosen": 0.19187921285629272,
      "logits/rejected": 0.2653835117816925,
      "logps/chosen": -580.5294189453125,
      "logps/rejected": -682.482421875,
      "loss": 0.2615,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.2366868555545807,
      "rewards/margins": 0.16842707991600037,
      "rewards/rejected": -0.40511393547058105,
      "step": 350
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.87284885712256e-06,
      "logits/chosen": 0.20482924580574036,
      "logits/rejected": 0.2829897999763489,
      "logps/chosen": -568.0645751953125,
      "logps/rejected": -742.5205078125,
      "loss": 0.231,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.23385734856128693,
      "rewards/margins": 0.20819666981697083,
      "rewards/rejected": -0.44205403327941895,
      "step": 360
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.857779879123181e-06,
      "logits/chosen": 0.1813650131225586,
      "logits/rejected": 0.28613919019699097,
      "logps/chosen": -620.7769775390625,
      "logps/rejected": -805.1932983398438,
      "loss": 0.2268,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2879635691642761,
      "rewards/margins": 0.2243933230638504,
      "rewards/rejected": -0.5123568773269653,
      "step": 370
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.841893264593643e-06,
      "logits/chosen": 0.1462257206439972,
      "logits/rejected": 0.2905661463737488,
      "logps/chosen": -616.0975341796875,
      "logps/rejected": -712.8366088867188,
      "loss": 0.2361,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26976972818374634,
      "rewards/margins": 0.16418993473052979,
      "rewards/rejected": -0.4339596629142761,
      "step": 380
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.825194522732023e-06,
      "logits/chosen": 0.21741768717765808,
      "logits/rejected": 0.28720468282699585,
      "logps/chosen": -553.0687255859375,
      "logps/rejected": -674.6417236328125,
      "loss": 0.2391,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.22814492881298065,
      "rewards/margins": 0.16390588879585266,
      "rewards/rejected": -0.3920508027076721,
      "step": 390
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.807689444367853e-06,
      "logits/chosen": 0.13199961185455322,
      "logits/rejected": 0.28842002153396606,
      "logps/chosen": -572.0911254882812,
      "logps/rejected": -669.98046875,
      "loss": 0.2511,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.24041178822517395,
      "rewards/margins": 0.1423041820526123,
      "rewards/rejected": -0.38271597027778625,
      "step": 400
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.78938409995396e-06,
      "logits/chosen": 0.23625163733959198,
      "logits/rejected": 0.3224749267101288,
      "logps/chosen": -669.3237915039062,
      "logps/rejected": -783.9891357421875,
      "loss": 0.2422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.316593199968338,
      "rewards/margins": 0.19081000983715057,
      "rewards/rejected": -0.5074031949043274,
      "step": 410
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.770284837461342e-06,
      "logits/chosen": 0.20555272698402405,
      "logits/rejected": 0.3196572959423065,
      "logps/chosen": -729.6986083984375,
      "logps/rejected": -817.2890625,
      "loss": 0.2429,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.341257780790329,
      "rewards/margins": 0.18218111991882324,
      "rewards/rejected": -0.5234388113021851,
      "step": 420
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7503982801778015e-06,
      "logits/chosen": 0.20105445384979248,
      "logits/rejected": 0.2788250744342804,
      "logps/chosen": -735.6785888671875,
      "logps/rejected": -862.8883056640625,
      "loss": 0.2052,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.3522084653377533,
      "rewards/margins": 0.2194521129131317,
      "rewards/rejected": -0.5716606378555298,
      "step": 430
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.729731324411104e-06,
      "logits/chosen": 0.1956392228603363,
      "logits/rejected": 0.3031262755393982,
      "logps/chosen": -678.7946166992188,
      "logps/rejected": -765.5419921875,
      "loss": 0.298,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.35866203904151917,
      "rewards/margins": 0.15328657627105713,
      "rewards/rejected": -0.5119485855102539,
      "step": 440
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.7082911370974645e-06,
      "logits/chosen": 0.17509567737579346,
      "logits/rejected": 0.27089959383010864,
      "logps/chosen": -666.20458984375,
      "logps/rejected": -816.4446411132812,
      "loss": 0.2403,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.31221580505371094,
      "rewards/margins": 0.20367129147052765,
      "rewards/rejected": -0.5158870816230774,
      "step": 450
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.68608515331618e-06,
      "logits/chosen": 0.17904916405677795,
      "logits/rejected": 0.20903074741363525,
      "logps/chosen": -623.48193359375,
      "logps/rejected": -798.4592895507812,
      "loss": 0.1872,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.28309115767478943,
      "rewards/margins": 0.22119691967964172,
      "rewards/rejected": -0.5042880773544312,
      "step": 460
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.663121073711269e-06,
      "logits/chosen": 0.17857761681079865,
      "logits/rejected": 0.22934532165527344,
      "logps/chosen": -671.294189453125,
      "logps/rejected": -820.5545654296875,
      "loss": 0.2177,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.340231329202652,
      "rewards/margins": 0.1799103319644928,
      "rewards/rejected": -0.5201416611671448,
      "step": 470
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.63940686182103e-06,
      "logits/chosen": 0.21654605865478516,
      "logits/rejected": 0.331270307302475,
      "logps/chosen": -630.5990600585938,
      "logps/rejected": -743.7600708007812,
      "loss": 0.2503,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.2956790328025818,
      "rewards/margins": 0.1739194393157959,
      "rewards/rejected": -0.4695984721183777,
      "step": 480
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.614950741316425e-06,
      "logits/chosen": 0.13878825306892395,
      "logits/rejected": 0.2916678488254547,
      "logps/chosen": -620.2591552734375,
      "logps/rejected": -782.7337646484375,
      "loss": 0.2216,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.31532377004623413,
      "rewards/margins": 0.2057572305202484,
      "rewards/rejected": -0.5210810303688049,
      "step": 490
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.589761193149254e-06,
      "logits/chosen": 0.17934757471084595,
      "logits/rejected": 0.2883814871311188,
      "logps/chosen": -649.7817993164062,
      "logps/rejected": -799.4210815429688,
      "loss": 0.2229,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3004361689090729,
      "rewards/margins": 0.2075958251953125,
      "rewards/rejected": -0.508031964302063,
      "step": 500
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.563846952611112e-06,
      "logits/chosen": 0.17266924679279327,
      "logits/rejected": 0.32525214552879333,
      "logps/chosen": -700.376220703125,
      "logps/rejected": -833.9622192382812,
      "loss": 0.2532,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.3490568995475769,
      "rewards/margins": 0.1844739019870758,
      "rewards/rejected": -0.5335308313369751,
      "step": 510
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.537217006304141e-06,
      "logits/chosen": 0.19843865931034088,
      "logits/rejected": 0.27735739946365356,
      "logps/chosen": -654.317626953125,
      "logps/rejected": -793.0462646484375,
      "loss": 0.2342,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.33207401633262634,
      "rewards/margins": 0.18460185825824738,
      "rewards/rejected": -0.5166758894920349,
      "step": 520
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.50988058902464e-06,
      "logits/chosen": 0.1701696366071701,
      "logits/rejected": 0.2598097026348114,
      "logps/chosen": -640.8900146484375,
      "logps/rejected": -793.0274658203125,
      "loss": 0.1985,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.31474485993385315,
      "rewards/margins": 0.19083499908447266,
      "rewards/rejected": -0.5055798292160034,
      "step": 530
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.481847180560593e-06,
      "logits/chosen": 0.232101172208786,
      "logits/rejected": 0.30248063802719116,
      "logps/chosen": -695.262451171875,
      "logps/rejected": -817.1578369140625,
      "loss": 0.2421,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.33937522768974304,
      "rewards/margins": 0.19629260897636414,
      "rewards/rejected": -0.5356678366661072,
      "step": 540
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.453126502404253e-06,
      "logits/chosen": 0.21882247924804688,
      "logits/rejected": 0.3331480920314789,
      "logps/chosen": -711.1233520507812,
      "logps/rejected": -841.3557739257812,
      "loss": 0.214,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3614684045314789,
      "rewards/margins": 0.19178052246570587,
      "rewards/rejected": -0.553248941898346,
      "step": 550
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.423728514380892e-06,
      "logits/chosen": 0.14314863085746765,
      "logits/rejected": 0.3489857614040375,
      "logps/chosen": -778.8663330078125,
      "logps/rejected": -877.1080932617188,
      "loss": 0.2251,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.4136313498020172,
      "rewards/margins": 0.19483810663223267,
      "rewards/rejected": -0.6084694266319275,
      "step": 560
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.393663411194918e-06,
      "logits/chosen": 0.19116893410682678,
      "logits/rejected": 0.32879945635795593,
      "logps/chosen": -710.8887939453125,
      "logps/rejected": -837.41748046875,
      "loss": 0.212,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.3603808283805847,
      "rewards/margins": 0.19663426280021667,
      "rewards/rejected": -0.557015061378479,
      "step": 570
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.362941618894523e-06,
      "logits/chosen": 0.14669467508792877,
      "logits/rejected": 0.3146809935569763,
      "logps/chosen": -662.716064453125,
      "logps/rejected": -765.9783935546875,
      "loss": 0.2587,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.32042309641838074,
      "rewards/margins": 0.1730494648218155,
      "rewards/rejected": -0.4934725761413574,
      "step": 580
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.331573791256116e-06,
      "logits/chosen": 0.171834796667099,
      "logits/rejected": 0.30899518728256226,
      "logps/chosen": -648.413330078125,
      "logps/rejected": -787.9014892578125,
      "loss": 0.195,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.29140716791152954,
      "rewards/margins": 0.20431244373321533,
      "rewards/rejected": -0.4957196116447449,
      "step": 590
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.299570806089786e-06,
      "logits/chosen": 0.1850585639476776,
      "logits/rejected": 0.3111046552658081,
      "logps/chosen": -640.8422241210938,
      "logps/rejected": -784.2683715820312,
      "loss": 0.2277,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.31103643774986267,
      "rewards/margins": 0.19471977651119232,
      "rewards/rejected": -0.5057561993598938,
      "step": 600
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.266943761467057e-06,
      "logits/chosen": 0.2157634198665619,
      "logits/rejected": 0.2784191071987152,
      "logps/chosen": -641.7967529296875,
      "logps/rejected": -809.9637451171875,
      "loss": 0.2216,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.30600252747535706,
      "rewards/margins": 0.2282377928495407,
      "rewards/rejected": -0.5342403650283813,
      "step": 610
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.233703971872287e-06,
      "logits/chosen": 0.17053952813148499,
      "logits/rejected": 0.3547263741493225,
      "logps/chosen": -663.431884765625,
      "logps/rejected": -784.136474609375,
      "loss": 0.2201,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.31853628158569336,
      "rewards/margins": 0.19354286789894104,
      "rewards/rejected": -0.512079119682312,
      "step": 620
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1998629642789925e-06,
      "logits/chosen": 0.11867985874414444,
      "logits/rejected": 0.3206768035888672,
      "logps/chosen": -657.0588989257812,
      "logps/rejected": -777.2775268554688,
      "loss": 0.2433,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.32349634170532227,
      "rewards/margins": 0.1930086761713028,
      "rewards/rejected": -0.5165050029754639,
      "step": 630
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.165432474152505e-06,
      "logits/chosen": 0.20680156350135803,
      "logits/rejected": 0.3131827116012573,
      "logps/chosen": -677.2208251953125,
      "logps/rejected": -788.533447265625,
      "loss": 0.2271,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.303560733795166,
      "rewards/margins": 0.18881794810295105,
      "rewards/rejected": -0.4923786520957947,
      "step": 640
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.130424441380308e-06,
      "logits/chosen": 0.17017784714698792,
      "logits/rejected": 0.2720898985862732,
      "logps/chosen": -604.7415161132812,
      "logps/rejected": -724.3334350585938,
      "loss": 0.2259,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.27009710669517517,
      "rewards/margins": 0.1792145073413849,
      "rewards/rejected": -0.44931164383888245,
      "step": 650
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.09485100613151e-06,
      "logits/chosen": 0.1530022919178009,
      "logits/rejected": 0.30241402983665466,
      "logps/chosen": -639.620361328125,
      "logps/rejected": -807.2040405273438,
      "loss": 0.214,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.2783825993537903,
      "rewards/margins": 0.229055255651474,
      "rewards/rejected": -0.5074378848075867,
      "step": 660
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.058724504646834e-06,
      "logits/chosen": 0.12180732190608978,
      "logits/rejected": 0.2914981245994568,
      "logps/chosen": -710.6832885742188,
      "logps/rejected": -850.0335083007812,
      "loss": 0.225,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.363064706325531,
      "rewards/margins": 0.19931283593177795,
      "rewards/rejected": -0.5623775720596313,
      "step": 670
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.022057464960632e-06,
      "logits/chosen": 0.17398427426815033,
      "logits/rejected": 0.3090857267379761,
      "logps/chosen": -684.0886840820312,
      "logps/rejected": -895.4011840820312,
      "loss": 0.1855,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3387846052646637,
      "rewards/margins": 0.2770841717720032,
      "rewards/rejected": -0.6158688068389893,
      "step": 680
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.984862602556383e-06,
      "logits/chosen": 0.20324131846427917,
      "logits/rejected": 0.30140548944473267,
      "logps/chosen": -664.1101684570312,
      "logps/rejected": -785.5363159179688,
      "loss": 0.2282,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.29074573516845703,
      "rewards/margins": 0.2039407193660736,
      "rewards/rejected": -0.4946863651275635,
      "step": 690
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.947152815957187e-06,
      "logits/chosen": 0.1982668787240982,
      "logits/rejected": 0.2695184648036957,
      "logps/chosen": -575.2810668945312,
      "logps/rejected": -737.6761474609375,
      "loss": 0.1907,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.25107094645500183,
      "rewards/margins": 0.1953134834766388,
      "rewards/rejected": -0.4463844895362854,
      "step": 700
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.908941182252785e-06,
      "logits/chosen": 0.17798396944999695,
      "logits/rejected": 0.32517749071121216,
      "logps/chosen": -664.4030151367188,
      "logps/rejected": -806.2379760742188,
      "loss": 0.2163,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2954537868499756,
      "rewards/margins": 0.22870102524757385,
      "rewards/rejected": -0.524154782295227,
      "step": 710
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.8702409525646535e-06,
      "logits/chosen": 0.23220598697662354,
      "logits/rejected": 0.3328869938850403,
      "logps/chosen": -631.212158203125,
      "logps/rejected": -779.985107421875,
      "loss": 0.2106,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.28802287578582764,
      "rewards/margins": 0.21716487407684326,
      "rewards/rejected": -0.5051878094673157,
      "step": 720
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8310655474507495e-06,
      "logits/chosen": 0.20683269202709198,
      "logits/rejected": 0.2652401924133301,
      "logps/chosen": -639.0568237304688,
      "logps/rejected": -799.6007690429688,
      "loss": 0.2347,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.29507094621658325,
      "rewards/margins": 0.2053174525499344,
      "rewards/rejected": -0.5003883838653564,
      "step": 730
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7914285522515002e-06,
      "logits/chosen": 0.19919940829277039,
      "logits/rejected": 0.3394211530685425,
      "logps/chosen": -686.8181762695312,
      "logps/rejected": -870.0245361328125,
      "loss": 0.2058,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.3031126856803894,
      "rewards/margins": 0.2735733389854431,
      "rewards/rejected": -0.5766860246658325,
      "step": 740
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.751343712378639e-06,
      "logits/chosen": 0.21238520741462708,
      "logits/rejected": 0.22055749595165253,
      "logps/chosen": -629.7884521484375,
      "logps/rejected": -798.6253662109375,
      "loss": 0.2525,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.31108587980270386,
      "rewards/margins": 0.18138930201530457,
      "rewards/rejected": -0.4924752116203308,
      "step": 750
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.710824928548546e-06,
      "logits/chosen": 0.16632458567619324,
      "logits/rejected": 0.2695624828338623,
      "logps/chosen": -619.755126953125,
      "logps/rejected": -766.7147216796875,
      "loss": 0.2216,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.283473938703537,
      "rewards/margins": 0.18939343094825745,
      "rewards/rejected": -0.4728673994541168,
      "step": 760
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6698862519617225e-06,
      "logits/chosen": 0.17467322945594788,
      "logits/rejected": 0.21252937614917755,
      "logps/chosen": -616.0049438476562,
      "logps/rejected": -769.6994018554688,
      "loss": 0.2496,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3090468943119049,
      "rewards/margins": 0.16970062255859375,
      "rewards/rejected": -0.47874751687049866,
      "step": 770
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6285418794300793e-06,
      "logits/chosen": 0.1817772388458252,
      "logits/rejected": 0.22469213604927063,
      "logps/chosen": -679.7005615234375,
      "logps/rejected": -825.8623046875,
      "loss": 0.215,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3420635461807251,
      "rewards/margins": 0.1958375722169876,
      "rewards/rejected": -0.5379011034965515,
      "step": 780
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5868061484537365e-06,
      "logits/chosen": 0.11898622661828995,
      "logits/rejected": 0.23143093287944794,
      "logps/chosen": -700.2951049804688,
      "logps/rejected": -797.0465087890625,
      "loss": 0.2019,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.33859121799468994,
      "rewards/margins": 0.1829417496919632,
      "rewards/rejected": -0.5215329527854919,
      "step": 790
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5446935322490285e-06,
      "logits/chosen": 0.09929606318473816,
      "logits/rejected": 0.23305658996105194,
      "logps/chosen": -629.5419311523438,
      "logps/rejected": -787.3818359375,
      "loss": 0.201,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.2896836996078491,
      "rewards/margins": 0.22416365146636963,
      "rewards/rejected": -0.5138473510742188,
      "step": 800
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.502218634729447e-06,
      "logits/chosen": 0.1264248788356781,
      "logits/rejected": 0.2334756851196289,
      "logps/chosen": -656.5744018554688,
      "logps/rejected": -850.2476806640625,
      "loss": 0.2062,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3103203773498535,
      "rewards/margins": 0.24082545936107635,
      "rewards/rejected": -0.5511458516120911,
      "step": 810
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.459396185441265e-06,
      "logits/chosen": 0.15474265813827515,
      "logits/rejected": 0.22842803597450256,
      "logps/chosen": -671.4266357421875,
      "logps/rejected": -916.7794799804688,
      "loss": 0.1971,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.31731024384498596,
      "rewards/margins": 0.28582102060317993,
      "rewards/rejected": -0.6031311750411987,
      "step": 820
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4162410344555834e-06,
      "logits/chosen": 0.13778944313526154,
      "logits/rejected": 0.27569061517715454,
      "logps/chosen": -631.9242553710938,
      "logps/rejected": -814.20068359375,
      "loss": 0.2123,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.31727999448776245,
      "rewards/margins": 0.21109680831432343,
      "rewards/rejected": -0.5283767580986023,
      "step": 830
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3727681472185937e-06,
      "logits/chosen": 0.09538077563047409,
      "logits/rejected": 0.2515682578086853,
      "logps/chosen": -641.342529296875,
      "logps/rejected": -826.2974853515625,
      "loss": 0.1821,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.26377955079078674,
      "rewards/margins": 0.26421433687210083,
      "rewards/rejected": -0.5279938578605652,
      "step": 840
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3289925993618217e-06,
      "logits/chosen": 0.1294173300266266,
      "logits/rejected": 0.27691131830215454,
      "logps/chosen": -666.8058471679688,
      "logps/rejected": -798.5427856445312,
      "loss": 0.2291,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.3166380226612091,
      "rewards/margins": 0.19483907520771027,
      "rewards/rejected": -0.5114771127700806,
      "step": 850
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2849295714741643e-06,
      "logits/chosen": 0.1251804232597351,
      "logits/rejected": 0.19463461637496948,
      "logps/chosen": -661.3643798828125,
      "logps/rejected": -868.736328125,
      "loss": 0.2382,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3124701678752899,
      "rewards/margins": 0.2407514601945877,
      "rewards/rejected": -0.553221583366394,
      "step": 860
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2405943438375287e-06,
      "logits/chosen": 0.1276322454214096,
      "logits/rejected": 0.18467645347118378,
      "logps/chosen": -618.6890869140625,
      "logps/rejected": -740.9835815429688,
      "loss": 0.2676,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.31308645009994507,
      "rewards/margins": 0.14988043904304504,
      "rewards/rejected": -0.4629668593406677,
      "step": 870
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1960022911279036e-06,
      "logits/chosen": 0.12038137763738632,
      "logits/rejected": 0.2056489735841751,
      "logps/chosen": -667.7818603515625,
      "logps/rejected": -825.8253173828125,
      "loss": 0.2025,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.303612619638443,
      "rewards/margins": 0.212577223777771,
      "rewards/rejected": -0.5161898136138916,
      "step": 880
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1511688770836844e-06,
      "logits/chosen": 0.1971738338470459,
      "logits/rejected": 0.2837975025177002,
      "logps/chosen": -673.0835571289062,
      "logps/rejected": -819.943359375,
      "loss": 0.1922,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.31264692544937134,
      "rewards/margins": 0.226339653134346,
      "rewards/rejected": -0.5389865636825562,
      "step": 890
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1061096491431307e-06,
      "logits/chosen": 0.15372741222381592,
      "logits/rejected": 0.2788582444190979,
      "logps/chosen": -612.5509033203125,
      "logps/rejected": -792.030517578125,
      "loss": 0.1956,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.30290934443473816,
      "rewards/margins": 0.2101304531097412,
      "rewards/rejected": -0.5130398273468018,
      "step": 900
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0608402330527796e-06,
      "logits/chosen": 0.1545824110507965,
      "logits/rejected": 0.25414803624153137,
      "logps/chosen": -599.8353271484375,
      "logps/rejected": -818.6397705078125,
      "loss": 0.1907,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.295954167842865,
      "rewards/margins": 0.24365074932575226,
      "rewards/rejected": -0.5396049618721008,
      "step": 910
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0153763274487176e-06,
      "logits/chosen": 0.18323026597499847,
      "logits/rejected": 0.3210206925868988,
      "logps/chosen": -620.0147705078125,
      "logps/rejected": -791.1729125976562,
      "loss": 0.2099,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2964519262313843,
      "rewards/margins": 0.2253306806087494,
      "rewards/rejected": -0.521782636642456,
      "step": 920
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9697336984125683e-06,
      "logits/chosen": 0.20417118072509766,
      "logits/rejected": 0.3416239619255066,
      "logps/chosen": -689.14697265625,
      "logps/rejected": -801.5291137695312,
      "loss": 0.256,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.3329721689224243,
      "rewards/margins": 0.19175252318382263,
      "rewards/rejected": -0.5247246026992798,
      "step": 930
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.923928174004094e-06,
      "logits/chosen": 0.16527384519577026,
      "logits/rejected": 0.3157365918159485,
      "logps/chosen": -678.1207885742188,
      "logps/rejected": -844.8040161132812,
      "loss": 0.2264,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3161105513572693,
      "rewards/margins": 0.22557704150676727,
      "rewards/rejected": -0.541687548160553,
      "step": 940
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8779756387723036e-06,
      "logits/chosen": 0.11534378677606583,
      "logits/rejected": 0.2540052831172943,
      "logps/chosen": -608.9468994140625,
      "logps/rejected": -772.9508666992188,
      "loss": 0.2025,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2806348502635956,
      "rewards/margins": 0.1979130059480667,
      "rewards/rejected": -0.4785478711128235,
      "step": 950
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.831892028246968e-06,
      "logits/chosen": 0.14051546156406403,
      "logits/rejected": 0.24735793471336365,
      "logps/chosen": -619.1126708984375,
      "logps/rejected": -770.658203125,
      "loss": 0.2095,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.27908432483673096,
      "rewards/margins": 0.2014855593442917,
      "rewards/rejected": -0.48056989908218384,
      "step": 960
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7856933234124617e-06,
      "logits/chosen": 0.16361235082149506,
      "logits/rejected": 0.2182026356458664,
      "logps/chosen": -574.9110107421875,
      "logps/rejected": -739.6251220703125,
      "loss": 0.1942,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.26594382524490356,
      "rewards/margins": 0.22089500725269318,
      "rewards/rejected": -0.48683881759643555,
      "step": 970
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7393955451658387e-06,
      "logits/chosen": 0.10398687422275543,
      "logits/rejected": 0.2140096127986908,
      "logps/chosen": -646.896240234375,
      "logps/rejected": -763.9075317382812,
      "loss": 0.2499,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.2980521321296692,
      "rewards/margins": 0.1578332483768463,
      "rewards/rejected": -0.4558853507041931,
      "step": 980
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6930147487610667e-06,
      "logits/chosen": 0.17513594031333923,
      "logits/rejected": 0.25838881731033325,
      "logps/chosen": -671.4928588867188,
      "logps/rejected": -827.3790893554688,
      "loss": 0.2148,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.30318453907966614,
      "rewards/margins": 0.2151859700679779,
      "rewards/rejected": -0.518370509147644,
      "step": 990
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6465670182413487e-06,
      "logits/chosen": 0.1189759150147438,
      "logits/rejected": 0.2538264989852905,
      "logps/chosen": -628.1451416015625,
      "logps/rejected": -789.3397827148438,
      "loss": 0.2139,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.27723228931427,
      "rewards/margins": 0.2113499641418457,
      "rewards/rejected": -0.48858222365379333,
      "step": 1000
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6000684608614594e-06,
      "logits/chosen": 0.15936598181724548,
      "logits/rejected": 0.300658255815506,
      "logps/chosen": -649.9871215820312,
      "logps/rejected": -798.48193359375,
      "loss": 0.2091,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.2968089282512665,
      "rewards/margins": 0.22261710464954376,
      "rewards/rejected": -0.5194259881973267,
      "step": 1010
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5535352015020338e-06,
      "logits/chosen": 0.14960740506649017,
      "logits/rejected": 0.24492999911308289,
      "logps/chosen": -646.4168701171875,
      "logps/rejected": -794.9764404296875,
      "loss": 0.2394,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3242790699005127,
      "rewards/margins": 0.1874123513698578,
      "rewards/rejected": -0.5116914510726929,
      "step": 1020
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.506983377077741e-06,
      "logits/chosen": 0.18958871066570282,
      "logits/rejected": 0.3043481707572937,
      "logps/chosen": -704.5410766601562,
      "logps/rejected": -824.1947021484375,
      "loss": 0.1955,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3319619297981262,
      "rewards/margins": 0.21646328270435333,
      "rewards/rejected": -0.5484252572059631,
      "step": 1030
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.460429130941289e-06,
      "logits/chosen": 0.12721054255962372,
      "logits/rejected": 0.2843805253505707,
      "logps/chosen": -730.2723388671875,
      "logps/rejected": -913.4192504882812,
      "loss": 0.1972,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.33351242542266846,
      "rewards/margins": 0.27772635221481323,
      "rewards/rejected": -0.6112387776374817,
      "step": 1040
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.413888607285192e-06,
      "logits/chosen": 0.13960017263889313,
      "logits/rejected": 0.27561983466148376,
      "logps/chosen": -659.7611083984375,
      "logps/rejected": -837.8316650390625,
      "loss": 0.2062,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.32992517948150635,
      "rewards/margins": 0.23987038433551788,
      "rewards/rejected": -0.569795548915863,
      "step": 1050
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.367377945543249e-06,
      "logits/chosen": 0.13057874143123627,
      "logits/rejected": 0.2790360152721405,
      "logps/chosen": -669.2300415039062,
      "logps/rejected": -898.046875,
      "loss": 0.1519,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3232869505882263,
      "rewards/margins": 0.28473716974258423,
      "rewards/rejected": -0.6080241799354553,
      "step": 1060
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.320913274793676e-06,
      "logits/chosen": 0.1361253708600998,
      "logits/rejected": 0.3236284852027893,
      "logps/chosen": -678.397216796875,
      "logps/rejected": -851.6448974609375,
      "loss": 0.2201,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3409735858440399,
      "rewards/margins": 0.22014662623405457,
      "rewards/rejected": -0.5611202120780945,
      "step": 1070
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.27451070816582e-06,
      "logits/chosen": 0.17740760743618011,
      "logits/rejected": 0.29873278737068176,
      "logps/chosen": -659.0814208984375,
      "logps/rejected": -794.0924682617188,
      "loss": 0.2546,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.3093605041503906,
      "rewards/margins": 0.18395116925239563,
      "rewards/rejected": -0.49331164360046387,
      "step": 1080
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.228186337252414e-06,
      "logits/chosen": 0.19297020137310028,
      "logits/rejected": 0.243372842669487,
      "logps/chosen": -609.9807739257812,
      "logps/rejected": -758.6661987304688,
      "loss": 0.2502,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.28266817331314087,
      "rewards/margins": 0.1889321208000183,
      "rewards/rejected": -0.4716002941131592,
      "step": 1090
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1819562265292946e-06,
      "logits/chosen": 0.1229313462972641,
      "logits/rejected": 0.24323606491088867,
      "logps/chosen": -608.5326538085938,
      "logps/rejected": -756.6920776367188,
      "loss": 0.2236,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.2758868932723999,
      "rewards/margins": 0.189192533493042,
      "rewards/rejected": -0.4650794565677643,
      "step": 1100
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1358364077845236e-06,
      "logits/chosen": 0.1506945639848709,
      "logits/rejected": 0.22459396719932556,
      "logps/chosen": -615.982177734375,
      "logps/rejected": -760.8124389648438,
      "loss": 0.2145,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.28815990686416626,
      "rewards/margins": 0.20497973263263702,
      "rewards/rejected": -0.4931396543979645,
      "step": 1110
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.089842874558849e-06,
      "logits/chosen": 0.15958169102668762,
      "logits/rejected": 0.2522016763687134,
      "logps/chosen": -692.9052124023438,
      "logps/rejected": -839.1974487304688,
      "loss": 0.2082,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.34207481145858765,
      "rewards/margins": 0.2148832082748413,
      "rewards/rejected": -0.556958019733429,
      "step": 1120
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0439915765994242e-06,
      "logits/chosen": 0.1427229940891266,
      "logits/rejected": 0.27327024936676025,
      "logps/chosen": -729.3465576171875,
      "logps/rejected": -863.1507568359375,
      "loss": 0.256,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.37482136487960815,
      "rewards/margins": 0.2067115306854248,
      "rewards/rejected": -0.5815330147743225,
      "step": 1130
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9982984143287186e-06,
      "logits/chosen": 0.13450102508068085,
      "logits/rejected": 0.2727622091770172,
      "logps/chosen": -695.5301513671875,
      "logps/rejected": -844.9054565429688,
      "loss": 0.2451,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.3660709857940674,
      "rewards/margins": 0.19018250703811646,
      "rewards/rejected": -0.5562534928321838,
      "step": 1140
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.95277923333053e-06,
      "logits/chosen": 0.1129334345459938,
      "logits/rejected": 0.24487635493278503,
      "logps/chosen": -764.1828002929688,
      "logps/rejected": -941.0518798828125,
      "loss": 0.1934,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.36524254083633423,
      "rewards/margins": 0.24579265713691711,
      "rewards/rejected": -0.6110352277755737,
      "step": 1150
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9074498188550156e-06,
      "logits/chosen": 0.06655772030353546,
      "logits/rejected": 0.20836982131004333,
      "logps/chosen": -686.903564453125,
      "logps/rejected": -835.3270263671875,
      "loss": 0.2293,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.3618778586387634,
      "rewards/margins": 0.20230205357074738,
      "rewards/rejected": -0.5641800165176392,
      "step": 1160
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.862325890344643e-06,
      "logits/chosen": 0.12320109456777573,
      "logits/rejected": 0.22472088038921356,
      "logps/chosen": -739.9735107421875,
      "logps/rejected": -892.1875,
      "loss": 0.211,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3738134503364563,
      "rewards/margins": 0.21076083183288574,
      "rewards/rejected": -0.5845743417739868,
      "step": 1170
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.817423095982972e-06,
      "logits/chosen": 0.12588045001029968,
      "logits/rejected": 0.23388127982616425,
      "logps/chosen": -743.3089599609375,
      "logps/rejected": -864.0367431640625,
      "loss": 0.2328,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.38337722420692444,
      "rewards/margins": 0.19610758125782013,
      "rewards/rejected": -0.579484760761261,
      "step": 1180
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7727570072681293e-06,
      "logits/chosen": 0.11823949962854385,
      "logits/rejected": 0.27891501784324646,
      "logps/chosen": -762.4296264648438,
      "logps/rejected": -890.8868408203125,
      "loss": 0.2148,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.3807776868343353,
      "rewards/margins": 0.21602845191955566,
      "rewards/rejected": -0.5968061685562134,
      "step": 1190
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7283431136128961e-06,
      "logits/chosen": 0.10199010372161865,
      "logits/rejected": 0.19605082273483276,
      "logps/chosen": -696.8466186523438,
      "logps/rejected": -917.4085693359375,
      "loss": 0.1802,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.331060528755188,
      "rewards/margins": 0.27726510167121887,
      "rewards/rejected": -0.6083256006240845,
      "step": 1200
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6841968169732478e-06,
      "logits/chosen": 0.09005758911371231,
      "logits/rejected": 0.19088336825370789,
      "logps/chosen": -639.28369140625,
      "logps/rejected": -782.0608520507812,
      "loss": 0.2276,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3297976553440094,
      "rewards/margins": 0.19218826293945312,
      "rewards/rejected": -0.5219858884811401,
      "step": 1210
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6403334265072284e-06,
      "logits/chosen": 0.11749686300754547,
      "logits/rejected": 0.2290586233139038,
      "logps/chosen": -687.2972412109375,
      "logps/rejected": -768.551025390625,
      "loss": 0.2672,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.33632606267929077,
      "rewards/margins": 0.16481277346611023,
      "rewards/rejected": -0.5011388063430786,
      "step": 1220
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5967681532660066e-06,
      "logits/chosen": 0.05939929559826851,
      "logits/rejected": 0.21132275462150574,
      "logps/chosen": -681.4169311523438,
      "logps/rejected": -855.4088134765625,
      "loss": 0.2162,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.32602256536483765,
      "rewards/margins": 0.2201872617006302,
      "rewards/rejected": -0.5462098121643066,
      "step": 1230
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5535161049189463e-06,
      "logits/chosen": 0.09077402204275131,
      "logits/rejected": 0.18115364015102386,
      "logps/chosen": -656.8944091796875,
      "logps/rejected": -790.2760009765625,
      "loss": 0.2539,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3007816672325134,
      "rewards/margins": 0.20733746886253357,
      "rewards/rejected": -0.5081192255020142,
      "step": 1240
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5105922805145356e-06,
      "logits/chosen": 0.07764784246683121,
      "logits/rejected": 0.2526463568210602,
      "logps/chosen": -716.9674682617188,
      "logps/rejected": -843.5106201171875,
      "loss": 0.2041,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.30957937240600586,
      "rewards/margins": 0.23023274540901184,
      "rewards/rejected": -0.5398121476173401,
      "step": 1250
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4680115652789823e-06,
      "logits/chosen": 0.13033337891101837,
      "logits/rejected": 0.21490690112113953,
      "logps/chosen": -674.5143432617188,
      "logps/rejected": -827.2547607421875,
      "loss": 0.212,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3340405523777008,
      "rewards/margins": 0.21128436923027039,
      "rewards/rejected": -0.5453249216079712,
      "step": 1260
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4257887254542767e-06,
      "logits/chosen": 0.0994940996170044,
      "logits/rejected": 0.23082152009010315,
      "logps/chosen": -656.1998291015625,
      "logps/rejected": -833.0138549804688,
      "loss": 0.2029,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3017929196357727,
      "rewards/margins": 0.23969045281410217,
      "rewards/rejected": -0.5414833426475525,
      "step": 1270
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3839384031775227e-06,
      "logits/chosen": 0.10936876386404037,
      "logits/rejected": 0.20990300178527832,
      "logps/chosen": -660.2184448242188,
      "logps/rejected": -846.3512573242188,
      "loss": 0.2101,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.31637948751449585,
      "rewards/margins": 0.23163414001464844,
      "rewards/rejected": -0.5480136871337891,
      "step": 1280
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.342475111403298e-06,
      "logits/chosen": 0.10218968242406845,
      "logits/rejected": 0.2423768788576126,
      "logps/chosen": -733.5655517578125,
      "logps/rejected": -864.4533081054688,
      "loss": 0.2025,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.324023574590683,
      "rewards/margins": 0.2365175038576126,
      "rewards/rejected": -0.560541033744812,
      "step": 1290
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3014132288708209e-06,
      "logits/chosen": 0.09106200188398361,
      "logits/rejected": 0.196526437997818,
      "logps/chosen": -663.4789428710938,
      "logps/rejected": -786.8779907226562,
      "loss": 0.2498,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.32671403884887695,
      "rewards/margins": 0.19609105587005615,
      "rewards/rejected": -0.5228050351142883,
      "step": 1300
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2607669951176549e-06,
      "logits/chosen": 0.09490621089935303,
      "logits/rejected": 0.26305851340293884,
      "logps/chosen": -669.8858032226562,
      "logps/rejected": -820.6624755859375,
      "loss": 0.1875,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.3120814263820648,
      "rewards/margins": 0.2310735285282135,
      "rewards/rejected": -0.5431549549102783,
      "step": 1310
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2205505055416891e-06,
      "logits/chosen": 0.07294271886348724,
      "logits/rejected": 0.15242712199687958,
      "logps/chosen": -652.7977294921875,
      "logps/rejected": -800.0413208007812,
      "loss": 0.2138,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.33722367882728577,
      "rewards/margins": 0.18947935104370117,
      "rewards/rejected": -0.5267030000686646,
      "step": 1320
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1807777065131002e-06,
      "logits/chosen": 0.07662687450647354,
      "logits/rejected": 0.206211119890213,
      "logps/chosen": -651.1556396484375,
      "logps/rejected": -795.70556640625,
      "loss": 0.2235,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.33270731568336487,
      "rewards/margins": 0.18210896849632263,
      "rewards/rejected": -0.5148162841796875,
      "step": 1330
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1414623905380012e-06,
      "logits/chosen": 0.10683353245258331,
      "logits/rejected": 0.14309810101985931,
      "logps/chosen": -733.0233154296875,
      "logps/rejected": -832.6646728515625,
      "loss": 0.2157,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.33933931589126587,
      "rewards/margins": 0.19576075673103333,
      "rewards/rejected": -0.5351000428199768,
      "step": 1340
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1026181914754388e-06,
      "logits/chosen": 0.045632436871528625,
      "logits/rejected": 0.22544965147972107,
      "logps/chosen": -701.60205078125,
      "logps/rejected": -801.7410888671875,
      "loss": 0.205,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3220959007740021,
      "rewards/margins": 0.21180233359336853,
      "rewards/rejected": -0.5338982343673706,
      "step": 1350
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0642585798094136e-06,
      "logits/chosen": 0.11129830032587051,
      "logits/rejected": 0.277961790561676,
      "logps/chosen": -685.46923828125,
      "logps/rejected": -835.8211059570312,
      "loss": 0.1954,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.3089851438999176,
      "rewards/margins": 0.24158485233783722,
      "rewards/rejected": -0.550570011138916,
      "step": 1360
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0263968579775522e-06,
      "logits/chosen": 0.08607203513383865,
      "logits/rejected": 0.2120324820280075,
      "logps/chosen": -670.0626831054688,
      "logps/rejected": -819.9489135742188,
      "loss": 0.2555,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3491436243057251,
      "rewards/margins": 0.190331369638443,
      "rewards/rejected": -0.5394749641418457,
      "step": 1370
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.89046155758058e-07,
      "logits/chosen": 0.09642196446657181,
      "logits/rejected": 0.21866998076438904,
      "logps/chosen": -670.3609619140625,
      "logps/rejected": -831.7306518554688,
      "loss": 0.1985,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.30979031324386597,
      "rewards/margins": 0.22851744294166565,
      "rewards/rejected": -0.538307785987854,
      "step": 1380
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.52219425716534e-07,
      "logits/chosen": 0.15417559444904327,
      "logits/rejected": 0.23215246200561523,
      "logps/chosen": -644.1177368164062,
      "logps/rejected": -855.5202026367188,
      "loss": 0.2107,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30276650190353394,
      "rewards/margins": 0.25079673528671265,
      "rewards/rejected": -0.5535632371902466,
      "step": 1390
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.15929438714262e-07,
      "logits/chosen": 0.1388302445411682,
      "logits/rejected": 0.2623920440673828,
      "logps/chosen": -750.0091552734375,
      "logps/rejected": -935.58251953125,
      "loss": 0.1996,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.36703410744667053,
      "rewards/margins": 0.24573786556720734,
      "rewards/rejected": -0.6127719283103943,
      "step": 1400
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.801887794794911e-07,
      "logits/chosen": 0.1403401494026184,
      "logits/rejected": 0.2397807389497757,
      "logps/chosen": -718.9532470703125,
      "logps/rejected": -856.7537231445312,
      "loss": 0.1984,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.32988426089286804,
      "rewards/margins": 0.23926320672035217,
      "rewards/rejected": -0.5691474676132202,
      "step": 1410
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.450098422432787e-07,
      "logits/chosen": 0.10472740232944489,
      "logits/rejected": 0.3000238537788391,
      "logps/chosen": -734.3883056640625,
      "logps/rejected": -850.7344970703125,
      "loss": 0.1807,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.341549813747406,
      "rewards/margins": 0.23151779174804688,
      "rewards/rejected": -0.5730676054954529,
      "step": 1420
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.104048264413858e-07,
      "logits/chosen": 0.16170555353164673,
      "logits/rejected": 0.23735301196575165,
      "logps/chosen": -682.0318603515625,
      "logps/rejected": -834.2679443359375,
      "loss": 0.2227,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.3349572718143463,
      "rewards/margins": 0.20067913830280304,
      "rewards/rejected": -0.5356364250183105,
      "step": 1430
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.763857324837321e-07,
      "logits/chosen": 0.09115556627511978,
      "logits/rejected": 0.20479066669940948,
      "logps/chosen": -661.0646362304688,
      "logps/rejected": -865.4320068359375,
      "loss": 0.2056,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.3348444104194641,
      "rewards/margins": 0.23586812615394592,
      "rewards/rejected": -0.5707124471664429,
      "step": 1440
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.429643575928605e-07,
      "logits/chosen": 0.09788443148136139,
      "logits/rejected": 0.24574396014213562,
      "logps/chosen": -734.4356689453125,
      "logps/rejected": -870.92236328125,
      "loss": 0.2197,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.35589176416397095,
      "rewards/margins": 0.2205926477909088,
      "rewards/rejected": -0.5764843225479126,
      "step": 1450
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.101522917128709e-07,
      "logits/chosen": 0.11591117084026337,
      "logits/rejected": 0.23913173377513885,
      "logps/chosen": -703.4747314453125,
      "logps/rejected": -862.8414916992188,
      "loss": 0.2322,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.3314746022224426,
      "rewards/margins": 0.23115992546081543,
      "rewards/rejected": -0.5626345872879028,
      "step": 1460
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.779609134902312e-07,
      "logits/chosen": 0.16051790118217468,
      "logits/rejected": 0.27909934520721436,
      "logps/chosen": -696.7191772460938,
      "logps/rejected": -821.4385986328125,
      "loss": 0.2326,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.3547061085700989,
      "rewards/margins": 0.1964879333972931,
      "rewards/rejected": -0.5511940717697144,
      "step": 1470
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.464013863278629e-07,
      "logits/chosen": 0.12144766002893448,
      "logits/rejected": 0.21441468596458435,
      "logps/chosen": -707.2329711914062,
      "logps/rejected": -870.8890380859375,
      "loss": 0.2213,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.34999531507492065,
      "rewards/margins": 0.2191038429737091,
      "rewards/rejected": -0.5690991282463074,
      "step": 1480
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.154846545138696e-07,
      "logits/chosen": 0.16252949833869934,
      "logits/rejected": 0.3306509852409363,
      "logps/chosen": -655.1351318359375,
      "logps/rejected": -821.1975708007812,
      "loss": 0.2245,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.33034515380859375,
      "rewards/margins": 0.22635062038898468,
      "rewards/rejected": -0.5566957592964172,
      "step": 1490
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.852214394262515e-07,
      "logits/chosen": 0.11046645790338516,
      "logits/rejected": 0.2029370814561844,
      "logps/chosen": -685.1182861328125,
      "logps/rejected": -852.0540161132812,
      "loss": 0.1796,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3330523669719696,
      "rewards/margins": 0.2372111976146698,
      "rewards/rejected": -0.5702635645866394,
      "step": 1500
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.556222358149191e-07,
      "logits/chosen": 0.13619574904441833,
      "logits/rejected": 0.21550631523132324,
      "logps/chosen": -650.5691528320312,
      "logps/rejected": -879.0435791015625,
      "loss": 0.2037,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3198038637638092,
      "rewards/margins": 0.2528200149536133,
      "rewards/rejected": -0.5726238489151001,
      "step": 1510
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.266973081622992e-07,
      "logits/chosen": 0.10330195724964142,
      "logits/rejected": 0.2254125326871872,
      "logps/chosen": -674.9971923828125,
      "logps/rejected": -825.3990478515625,
      "loss": 0.2224,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.3395020067691803,
      "rewards/margins": 0.21332807838916779,
      "rewards/rejected": -0.5528300404548645,
      "step": 1520
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.984566871237942e-07,
      "logits/chosen": 0.142780140042305,
      "logits/rejected": 0.23002369701862335,
      "logps/chosen": -725.0726318359375,
      "logps/rejected": -843.7293701171875,
      "loss": 0.2347,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.3490670323371887,
      "rewards/margins": 0.21415701508522034,
      "rewards/rejected": -0.5632239580154419,
      "step": 1530
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.709101660493251e-07,
      "logits/chosen": 0.1431182622909546,
      "logits/rejected": 0.2636813521385193,
      "logps/chosen": -640.8516845703125,
      "logps/rejected": -807.3698120117188,
      "loss": 0.2157,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.3060576617717743,
      "rewards/margins": 0.24311813712120056,
      "rewards/rejected": -0.5491757988929749,
      "step": 1540
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.440672975871743e-07,
      "logits/chosen": 0.09661155939102173,
      "logits/rejected": 0.21029344201087952,
      "logps/chosen": -712.8273315429688,
      "logps/rejected": -845.70263671875,
      "loss": 0.2374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32992398738861084,
      "rewards/margins": 0.21484914422035217,
      "rewards/rejected": -0.5447731018066406,
      "step": 1550
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1793739037129134e-07,
      "logits/chosen": 0.1100638136267662,
      "logits/rejected": 0.25361213088035583,
      "logps/chosen": -619.2400512695312,
      "logps/rejected": -786.0677490234375,
      "loss": 0.2424,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3038698732852936,
      "rewards/margins": 0.20768532156944275,
      "rewards/rejected": -0.5115551948547363,
      "step": 1560
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9252950579322405e-07,
      "logits/chosen": 0.1189003735780716,
      "logits/rejected": 0.21380853652954102,
      "logps/chosen": -671.5353393554688,
      "logps/rejected": -847.8660888671875,
      "loss": 0.2415,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.34101420640945435,
      "rewards/margins": 0.20406432449817657,
      "rewards/rejected": -0.545078456401825,
      "step": 1570
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.6785245485978864e-07,
      "logits/chosen": 0.10434021800756454,
      "logits/rejected": 0.2150150090456009,
      "logps/chosen": -659.5474853515625,
      "logps/rejected": -801.54052734375,
      "loss": 0.2019,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28841426968574524,
      "rewards/margins": 0.23222117125988007,
      "rewards/rejected": -0.5206354856491089,
      "step": 1580
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.43914795137566e-07,
      "logits/chosen": 0.13257619738578796,
      "logits/rejected": 0.2614479959011078,
      "logps/chosen": -638.5202026367188,
      "logps/rejected": -793.9677734375,
      "loss": 0.1989,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.29665178060531616,
      "rewards/margins": 0.21199484169483185,
      "rewards/rejected": -0.5086466073989868,
      "step": 1590
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.207248277852901e-07,
      "logits/chosen": 0.10789693892002106,
      "logits/rejected": 0.24690580368041992,
      "logps/chosen": -667.6051025390625,
      "logps/rejected": -793.3514404296875,
      "loss": 0.2256,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.3308831751346588,
      "rewards/margins": 0.1889718472957611,
      "rewards/rejected": -0.5198550224304199,
      "step": 1600
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.9829059467515074e-07,
      "logits/chosen": 0.052944671362638474,
      "logits/rejected": 0.1601809859275818,
      "logps/chosen": -652.5521240234375,
      "logps/rejected": -823.5758666992188,
      "loss": 0.2115,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.32761263847351074,
      "rewards/margins": 0.21856379508972168,
      "rewards/rejected": -0.5461763739585876,
      "step": 1610
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.766198756040153e-07,
      "logits/chosen": 0.10616923868656158,
      "logits/rejected": 0.2180422991514206,
      "logps/chosen": -636.3297119140625,
      "logps/rejected": -800.4964599609375,
      "loss": 0.203,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.303327739238739,
      "rewards/margins": 0.21036402881145477,
      "rewards/rejected": -0.513691782951355,
      "step": 1620
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5572018559553155e-07,
      "logits/chosen": 0.09138272702693939,
      "logits/rejected": 0.20507724583148956,
      "logps/chosen": -648.2923583984375,
      "logps/rejected": -802.4161987304688,
      "loss": 0.2113,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3106837868690491,
      "rewards/margins": 0.21935255825519562,
      "rewards/rejected": -0.5300363302230835,
      "step": 1630
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3559877229404864e-07,
      "logits/chosen": 0.07225445657968521,
      "logits/rejected": 0.2319096028804779,
      "logps/chosen": -682.279296875,
      "logps/rejected": -819.6962890625,
      "loss": 0.2117,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.32190632820129395,
      "rewards/margins": 0.2106344997882843,
      "rewards/rejected": -0.5325408577919006,
      "step": 1640
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1626261345126576e-07,
      "logits/chosen": 0.09080176055431366,
      "logits/rejected": 0.22487780451774597,
      "logps/chosen": -728.700439453125,
      "logps/rejected": -872.6180419921875,
      "loss": 0.2372,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35785117745399475,
      "rewards/margins": 0.20169305801391602,
      "rewards/rejected": -0.5595442056655884,
      "step": 1650
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9771841450646505e-07,
      "logits/chosen": 0.11907955259084702,
      "logits/rejected": 0.2156171053647995,
      "logps/chosen": -644.99560546875,
      "logps/rejected": -801.831787109375,
      "loss": 0.2024,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.308158278465271,
      "rewards/margins": 0.2264772653579712,
      "rewards/rejected": -0.5346355438232422,
      "step": 1660
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7997260626118758e-07,
      "logits/chosen": 0.12947291135787964,
      "logits/rejected": 0.22007863223552704,
      "logps/chosen": -710.0556030273438,
      "logps/rejected": -859.5272216796875,
      "loss": 0.1906,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.3359071612358093,
      "rewards/margins": 0.237164705991745,
      "rewards/rejected": -0.5730718374252319,
      "step": 1670
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6303134264914365e-07,
      "logits/chosen": 0.09690668433904648,
      "logits/rejected": 0.19589988887310028,
      "logps/chosen": -685.9202880859375,
      "logps/rejected": -816.6348876953125,
      "loss": 0.2104,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.3336608111858368,
      "rewards/margins": 0.20780424773693085,
      "rewards/rejected": -0.5414651036262512,
      "step": 1680
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.469004986021355e-07,
      "logits/chosen": 0.10044785588979721,
      "logits/rejected": 0.1938716471195221,
      "logps/chosen": -668.4989013671875,
      "logps/rejected": -797.7630615234375,
      "loss": 0.2075,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.3294285535812378,
      "rewards/margins": 0.20013563334941864,
      "rewards/rejected": -0.5295641422271729,
      "step": 1690
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.315856680127367e-07,
      "logits/chosen": 0.11922971159219742,
      "logits/rejected": 0.1923786997795105,
      "logps/chosen": -688.7864990234375,
      "logps/rejected": -845.6558837890625,
      "loss": 0.2053,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.34360042214393616,
      "rewards/margins": 0.2216511070728302,
      "rewards/rejected": -0.5652515292167664,
      "step": 1700
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1709216179442817e-07,
      "logits/chosen": 0.0872936099767685,
      "logits/rejected": 0.1970546394586563,
      "logps/chosen": -678.56982421875,
      "logps/rejected": -870.0894775390625,
      "loss": 0.2064,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.32151633501052856,
      "rewards/margins": 0.24507319927215576,
      "rewards/rejected": -0.5665895342826843,
      "step": 1710
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0342500603986421e-07,
      "logits/chosen": 0.09590867161750793,
      "logits/rejected": 0.20923268795013428,
      "logps/chosen": -639.2576904296875,
      "logps/rejected": -808.9605712890625,
      "loss": 0.191,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.3116968870162964,
      "rewards/margins": 0.22501316666603088,
      "rewards/rejected": -0.5367100834846497,
      "step": 1720
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.058894027791643e-08,
      "logits/chosen": 0.10010067373514175,
      "logits/rejected": 0.24267680943012238,
      "logps/chosen": -634.7396850585938,
      "logps/rejected": -830.8876953125,
      "loss": 0.2178,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.29997509717941284,
      "rewards/margins": 0.24514777958393097,
      "rewards/rejected": -0.5451229214668274,
      "step": 1730
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.858841583008592e-08,
      "logits/chosen": 0.08919323980808258,
      "logits/rejected": 0.17960329353809357,
      "logps/chosen": -699.3638916015625,
      "logps/rejected": -878.31689453125,
      "loss": 0.1988,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.32689014077186584,
      "rewards/margins": 0.23287110030651093,
      "rewards/rejected": -0.5597612857818604,
      "step": 1740
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.742759426686313e-08,
      "logits/chosen": 0.07647444307804108,
      "logits/rejected": 0.25179851055145264,
      "logps/chosen": -693.7523803710938,
      "logps/rejected": -840.65234375,
      "loss": 0.2016,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.3338211476802826,
      "rewards/margins": 0.21626095473766327,
      "rewards/rejected": -0.5500820875167847,
      "step": 1750
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.7110345964571104e-08,
      "logits/chosen": 0.05137849971652031,
      "logits/rejected": 0.19751080870628357,
      "logps/chosen": -635.8048095703125,
      "logps/rejected": -825.5013427734375,
      "loss": 0.212,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.31089353561401367,
      "rewards/margins": 0.23858308792114258,
      "rewards/rejected": -0.5494766235351562,
      "step": 1760
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.764024876318357e-08,
      "logits/chosen": 0.13479098677635193,
      "logits/rejected": 0.22830185294151306,
      "logps/chosen": -664.4447631835938,
      "logps/rejected": -820.0628662109375,
      "loss": 0.2085,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.2986433804035187,
      "rewards/margins": 0.2351032793521881,
      "rewards/rejected": -0.533746600151062,
      "step": 1770
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.902058672559633e-08,
      "logits/chosen": 0.0736847072839737,
      "logits/rejected": 0.21154184639453888,
      "logps/chosen": -644.4476928710938,
      "logps/rejected": -827.611328125,
      "loss": 0.2101,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.32568278908729553,
      "rewards/margins": 0.22031357884407043,
      "rewards/rejected": -0.5459963083267212,
      "step": 1780
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.125434899876933e-08,
      "logits/chosen": 0.11332382261753082,
      "logits/rejected": 0.2286304533481598,
      "logps/chosen": -655.7117919921875,
      "logps/rejected": -838.4694213867188,
      "loss": 0.2403,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.3177303373813629,
      "rewards/margins": 0.2210434377193451,
      "rewards/rejected": -0.538773775100708,
      "step": 1790
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4344228777145873e-08,
      "logits/chosen": 0.09552569687366486,
      "logits/rejected": 0.21039338409900665,
      "logps/chosen": -625.5585327148438,
      "logps/rejected": -785.0277099609375,
      "loss": 0.2167,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3133324980735779,
      "rewards/margins": 0.19650213420391083,
      "rewards/rejected": -0.5098346471786499,
      "step": 1800
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.829262236869772e-08,
      "logits/chosen": 0.12665894627571106,
      "logits/rejected": 0.25432324409484863,
      "logps/chosen": -717.9019775390625,
      "logps/rejected": -885.8001098632812,
      "loss": 0.181,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.34058862924575806,
      "rewards/margins": 0.23419833183288574,
      "rewards/rejected": -0.5747869610786438,
      "step": 1810
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.3101628363929586e-08,
      "logits/chosen": 0.07009974867105484,
      "logits/rejected": 0.18696331977844238,
      "logps/chosen": -648.749267578125,
      "logps/rejected": -838.9640502929688,
      "loss": 0.2079,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.30777597427368164,
      "rewards/margins": 0.23736786842346191,
      "rewards/rejected": -0.5451437830924988,
      "step": 1820
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.773046908123195e-09,
      "logits/chosen": 0.04345223680138588,
      "logits/rejected": 0.21128901839256287,
      "logps/chosen": -680.11376953125,
      "logps/rejected": -825.4275512695312,
      "loss": 0.2032,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3200992941856384,
      "rewards/margins": 0.22688953578472137,
      "rewards/rejected": -0.546988844871521,
      "step": 1830
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.308379077080817e-09,
      "logits/chosen": 0.11998043209314346,
      "logits/rejected": 0.20321743190288544,
      "logps/chosen": -673.5576782226562,
      "logps/rejected": -804.1566162109375,
      "loss": 0.2202,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.32543522119522095,
      "rewards/margins": 0.22123821079730988,
      "rewards/rejected": -0.5466734170913696,
      "step": 1840
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.7088263565760996e-09,
      "logits/chosen": 0.08479473739862442,
      "logits/rejected": 0.24488599598407745,
      "logps/chosen": -624.8433837890625,
      "logps/rejected": -800.512939453125,
      "loss": 0.2102,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3155784010887146,
      "rewards/margins": 0.21801578998565674,
      "rewards/rejected": -0.5335941910743713,
      "step": 1850
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.752902257023633e-10,
      "logits/chosen": 0.0935547798871994,
      "logits/rejected": 0.16574546694755554,
      "logps/chosen": -698.1727294921875,
      "logps/rejected": -887.7645263671875,
      "loss": 0.214,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.36060822010040283,
      "rewards/margins": 0.218820258975029,
      "rewards/rejected": -0.5794284343719482,
      "step": 1860
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.083718442532189e-10,
      "logits/chosen": 0.07583248615264893,
      "logits/rejected": 0.24347925186157227,
      "logps/chosen": -675.1774291992188,
      "logps/rejected": -852.7301025390625,
      "loss": 0.2069,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.3129870295524597,
      "rewards/margins": 0.24173569679260254,
      "rewards/rejected": -0.5547226667404175,
      "step": 1870
    },
    {
      "epoch": 1.0,
      "step": 1875,
      "total_flos": 0.0,
      "train_loss": 0.23111479535102844,
      "train_runtime": 16991.5507,
      "train_samples_per_second": 1.766,
      "train_steps_per_second": 0.11
    }
  ],
  "logging_steps": 10,
  "max_steps": 1875,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}