{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 1724,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 0.43359375,
      "learning_rate": 2.890173410404624e-09,
      "logits/chosen": 0.1325806975364685,
      "logits/rejected": 0.3077998757362366,
      "logps/chosen": -239.35935974121094,
      "logps/rejected": -304.581298828125,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/margins_max": 0.0,
      "rewards/margins_min": 0.0,
      "rewards/margins_std": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "grad_norm": 0.416015625,
      "learning_rate": 2.890173410404624e-08,
      "logits/chosen": -0.010774746537208557,
      "logits/rejected": 0.23452165722846985,
      "logps/chosen": -243.3074493408203,
      "logps/rejected": -304.1199035644531,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00028879166347905993,
      "rewards/margins": 0.0006378353573381901,
      "rewards/margins_max": 0.0028404404874891043,
      "rewards/margins_min": -0.0015647696563974023,
      "rewards/margins_std": 0.0031149541027843952,
      "rewards/rejected": -0.00034904375206679106,
      "step": 10
    },
    {
      "epoch": 0.01,
      "grad_norm": 0.400390625,
      "learning_rate": 5.780346820809248e-08,
      "logits/chosen": -0.05719061568379402,
      "logits/rejected": 0.5148837566375732,
      "logps/chosen": -272.7169494628906,
      "logps/rejected": -216.58859252929688,
      "loss": 0.6931,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0008704366046003997,
      "rewards/margins": 0.0001740378502290696,
      "rewards/margins_max": 0.0022189407609403133,
      "rewards/margins_min": -0.0018708650022745132,
      "rewards/margins_std": 0.002891929354518652,
      "rewards/rejected": -0.0010444745421409607,
      "step": 20
    },
    {
      "epoch": 0.02,
      "grad_norm": 0.4921875,
      "learning_rate": 8.670520231213872e-08,
      "logits/chosen": 0.05507341027259827,
      "logits/rejected": 0.5646872520446777,
      "logps/chosen": -272.96728515625,
      "logps/rejected": -252.10733032226562,
      "loss": 0.6932,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0014279346214607358,
      "rewards/margins": -0.001033178297802806,
      "rewards/margins_max": 0.002007028553634882,
      "rewards/margins_min": -0.004073385149240494,
      "rewards/margins_std": 0.00429950188845396,
      "rewards/rejected": -0.00039475635276176035,
      "step": 30
    },
    {
      "epoch": 0.02,
      "grad_norm": 0.447265625,
      "learning_rate": 1.1560693641618496e-07,
      "logits/chosen": -0.08530770242214203,
      "logits/rejected": 0.37523841857910156,
      "logps/chosen": -256.03692626953125,
      "logps/rejected": -224.8648223876953,
      "loss": 0.6932,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.0013576907804235816,
      "rewards/margins": -0.0014004515251144767,
      "rewards/margins_max": 0.0015217246254906058,
      "rewards/margins_min": -0.004322628024965525,
      "rewards/margins_std": 0.0041325814090669155,
      "rewards/rejected": 4.276079198461957e-05,
      "step": 40
    },
    {
      "epoch": 0.03,
      "grad_norm": 0.45703125,
      "learning_rate": 1.445086705202312e-07,
      "logits/chosen": 0.10976707935333252,
      "logits/rejected": 0.40187758207321167,
      "logps/chosen": -205.61318969726562,
      "logps/rejected": -214.9802703857422,
      "loss": 0.693,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0007841205224394798,
      "rewards/margins": 0.0018329259473830462,
      "rewards/margins_max": 0.004336017183959484,
      "rewards/margins_min": -0.0006701658712700009,
      "rewards/margins_std": 0.0035399063490331173,
      "rewards/rejected": -0.0010488051921129227,
      "step": 50
    },
    {
      "epoch": 0.03,
      "grad_norm": 0.39453125,
      "learning_rate": 1.7341040462427744e-07,
      "logits/chosen": 0.2901094853878021,
      "logits/rejected": 0.4794164299964905,
      "logps/chosen": -207.44509887695312,
      "logps/rejected": -231.39382934570312,
      "loss": 0.693,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.001270442851819098,
      "rewards/margins": -0.0007280521094799042,
      "rewards/margins_max": 0.0019893264397978783,
      "rewards/margins_min": -0.0034454308915883303,
      "rewards/margins_std": 0.0038429535925388336,
      "rewards/rejected": -0.0005423908005468547,
      "step": 60
    },
    {
      "epoch": 0.04,
      "grad_norm": 0.435546875,
      "learning_rate": 2.023121387283237e-07,
      "logits/chosen": 0.035371266305446625,
      "logits/rejected": 0.4755796492099762,
      "logps/chosen": -259.833740234375,
      "logps/rejected": -226.2167205810547,
      "loss": 0.6929,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0010710505302995443,
      "rewards/margins": 0.0011786860413849354,
      "rewards/margins_max": 0.004792899824678898,
      "rewards/margins_min": -0.002435527741909027,
      "rewards/margins_std": 0.005111270118504763,
      "rewards/rejected": -0.0022497368045151234,
      "step": 70
    },
    {
      "epoch": 0.05,
      "grad_norm": 0.4609375,
      "learning_rate": 2.3121387283236991e-07,
      "logits/chosen": 0.27303510904312134,
      "logits/rejected": 0.7382463216781616,
      "logps/chosen": -217.78671264648438,
      "logps/rejected": -208.35910034179688,
      "loss": 0.6928,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.2639263988821767e-05,
      "rewards/margins": 0.0014770211419090629,
      "rewards/margins_max": 0.0042491876520216465,
      "rewards/margins_min": -0.0012951450189575553,
      "rewards/margins_std": 0.003920434974133968,
      "rewards/rejected": -0.0014996604295447469,
      "step": 80
    },
    {
      "epoch": 0.05,
      "grad_norm": 0.6640625,
      "learning_rate": 2.601156069364162e-07,
      "logits/chosen": -0.20650863647460938,
      "logits/rejected": 0.17405006289482117,
      "logps/chosen": -226.12808227539062,
      "logps/rejected": -233.56381225585938,
      "loss": 0.692,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.000633719377219677,
      "rewards/margins": 0.0017947215819731355,
      "rewards/margins_max": 0.004501459188759327,
      "rewards/margins_min": -0.0009120159666053951,
      "rewards/margins_std": 0.0038279048167169094,
      "rewards/rejected": -0.0011610020883381367,
      "step": 90
    },
    {
      "epoch": 0.06,
      "grad_norm": 0.431640625,
      "learning_rate": 2.890173410404624e-07,
      "logits/chosen": -0.019260473549365997,
      "logits/rejected": 0.5504380464553833,
      "logps/chosen": -292.51995849609375,
      "logps/rejected": -235.86843872070312,
      "loss": 0.6919,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.001650218851864338,
      "rewards/margins": 0.002649242291226983,
      "rewards/margins_max": 0.005218566861003637,
      "rewards/margins_min": 7.99179106252268e-05,
      "rewards/margins_std": 0.0036335731856524944,
      "rewards/rejected": -0.0009990233229473233,
      "step": 100
    },
    {
      "epoch": 0.06,
      "grad_norm": 0.53125,
      "learning_rate": 3.1791907514450865e-07,
      "logits/chosen": -0.06840448081493378,
      "logits/rejected": 0.6899427175521851,
      "logps/chosen": -252.0308380126953,
      "logps/rejected": -199.84799194335938,
      "loss": 0.6918,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0018273231107741594,
      "rewards/margins": 0.00415054801851511,
      "rewards/margins_max": 0.0076604606583714485,
      "rewards/margins_min": 0.0006406344473361969,
      "rewards/margins_std": 0.004963767249137163,
      "rewards/rejected": -0.0023232249077409506,
      "step": 110
    },
    {
      "epoch": 0.07,
      "grad_norm": 0.36328125,
      "learning_rate": 3.468208092485549e-07,
      "logits/chosen": 0.09203040599822998,
      "logits/rejected": 0.5125548243522644,
      "logps/chosen": -256.213623046875,
      "logps/rejected": -232.49942016601562,
      "loss": 0.6915,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0007183876005001366,
      "rewards/margins": 0.004233072511851788,
      "rewards/margins_max": 0.007029411382973194,
      "rewards/margins_min": 0.0014367332914844155,
      "rewards/margins_std": 0.003954620566219091,
      "rewards/rejected": -0.0035146852023899555,
      "step": 120
    },
    {
      "epoch": 0.08,
      "grad_norm": 0.462890625,
      "learning_rate": 3.757225433526011e-07,
      "logits/chosen": -0.027632858604192734,
      "logits/rejected": 0.39557844400405884,
      "logps/chosen": -266.2771911621094,
      "logps/rejected": -271.76116943359375,
      "loss": 0.6907,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.002352924318984151,
      "rewards/margins": 0.005208231043070555,
      "rewards/margins_max": 0.008825947530567646,
      "rewards/margins_min": 0.001590514904819429,
      "rewards/margins_std": 0.005116222891956568,
      "rewards/rejected": -0.00285530649125576,
      "step": 130
    },
    {
      "epoch": 0.08,
      "grad_norm": 0.40625,
      "learning_rate": 4.046242774566474e-07,
      "logits/chosen": 0.06764040887355804,
      "logits/rejected": 0.3966519236564636,
      "logps/chosen": -178.83749389648438,
      "logps/rejected": -188.39877319335938,
      "loss": 0.6908,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.0029165446758270264,
      "rewards/margins": 0.006306161172688007,
      "rewards/margins_max": 0.009462257847189903,
      "rewards/margins_min": 0.0031500644981861115,
      "rewards/margins_std": 0.004463394172489643,
      "rewards/rejected": -0.0033896160311996937,
      "step": 140
    },
    {
      "epoch": 0.09,
      "grad_norm": 0.447265625,
      "learning_rate": 4.3352601156069365e-07,
      "logits/chosen": 0.011811649426817894,
      "logits/rejected": 0.4984157979488373,
      "logps/chosen": -268.1231994628906,
      "logps/rejected": -223.78799438476562,
      "loss": 0.6899,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.002369340742006898,
      "rewards/margins": 0.006674068979918957,
      "rewards/margins_max": 0.013764929957687855,
      "rewards/margins_min": -0.0004167918232269585,
      "rewards/margins_std": 0.010027991607785225,
      "rewards/rejected": -0.0043047284707427025,
      "step": 150
    },
    {
      "epoch": 0.09,
      "grad_norm": 0.322265625,
      "learning_rate": 4.6242774566473983e-07,
      "logits/chosen": -0.03828499838709831,
      "logits/rejected": 0.3794795870780945,
      "logps/chosen": -245.52865600585938,
      "logps/rejected": -234.1727752685547,
      "loss": 0.689,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.004552280530333519,
      "rewards/margins": 0.008487861603498459,
      "rewards/margins_max": 0.012918056920170784,
      "rewards/margins_min": 0.004057666752487421,
      "rewards/margins_std": 0.006265241652727127,
      "rewards/rejected": -0.003935581538826227,
      "step": 160
    },
    {
      "epoch": 0.1,
      "grad_norm": 0.49609375,
      "learning_rate": 4.913294797687861e-07,
      "logits/chosen": -0.0168992280960083,
      "logits/rejected": 0.500325620174408,
      "logps/chosen": -296.49517822265625,
      "logps/rejected": -248.3328094482422,
      "loss": 0.6887,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.003083079354837537,
      "rewards/margins": 0.006065175868570805,
      "rewards/margins_max": 0.011483820155262947,
      "rewards/margins_min": 0.0006465300684794784,
      "rewards/margins_std": 0.0076631223782896996,
      "rewards/rejected": -0.002982096979394555,
      "step": 170
    },
    {
      "epoch": 0.1,
      "grad_norm": 0.40625,
      "learning_rate": 4.999748710138438e-07,
      "logits/chosen": 0.14815935492515564,
      "logits/rejected": 0.5510139465332031,
      "logps/chosen": -233.9811553955078,
      "logps/rejected": -228.5449676513672,
      "loss": 0.688,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.003167560789734125,
      "rewards/margins": 0.007796141318976879,
      "rewards/margins_max": 0.012642833404242992,
      "rewards/margins_min": 0.002949449699372053,
      "rewards/margins_std": 0.006854257546365261,
      "rewards/rejected": -0.004628580994904041,
      "step": 180
    },
    {
      "epoch": 0.11,
      "grad_norm": 0.416015625,
      "learning_rate": 4.998518024263461e-07,
      "logits/chosen": 0.19040322303771973,
      "logits/rejected": 0.6236617565155029,
      "logps/chosen": -230.96762084960938,
      "logps/rejected": -211.4745330810547,
      "loss": 0.6871,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.006373309530317783,
      "rewards/margins": 0.012960617430508137,
      "rewards/margins_max": 0.01996336504817009,
      "rewards/margins_min": 0.0059578740037977695,
      "rewards/margins_std": 0.0099033759906888,
      "rewards/rejected": -0.006587309297174215,
      "step": 190
    },
    {
      "epoch": 0.12,
      "grad_norm": 0.416015625,
      "learning_rate": 4.996262291366814e-07,
      "logits/chosen": 0.054732900112867355,
      "logits/rejected": 0.22424785792827606,
      "logps/chosen": -210.0012664794922,
      "logps/rejected": -233.76388549804688,
      "loss": 0.6873,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.004412280861288309,
      "rewards/margins": 0.011961949989199638,
      "rewards/margins_max": 0.017657486721873283,
      "rewards/margins_min": 0.006266415119171143,
      "rewards/margins_std": 0.0080547034740448,
      "rewards/rejected": -0.007549669593572617,
      "step": 200
    },
    {
      "epoch": 0.12,
      "grad_norm": 0.498046875,
      "learning_rate": 4.992982436890003e-07,
      "logits/chosen": 0.09016792476177216,
      "logits/rejected": 0.45956069231033325,
      "logps/chosen": -226.3985595703125,
      "logps/rejected": -221.092529296875,
      "loss": 0.6868,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.005489318631589413,
      "rewards/margins": 0.013238553889095783,
      "rewards/margins_max": 0.018587926402688026,
      "rewards/margins_min": 0.00788918323814869,
      "rewards/margins_std": 0.007565152831375599,
      "rewards/rejected": -0.007749234326183796,
      "step": 210
    },
    {
      "epoch": 0.13,
      "grad_norm": 0.458984375,
      "learning_rate": 4.988679806432711e-07,
      "logits/chosen": -0.08951343595981598,
      "logits/rejected": 0.46994414925575256,
      "logps/chosen": -264.4379577636719,
      "logps/rejected": -236.77346801757812,
      "loss": 0.6853,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.007678179536014795,
      "rewards/margins": 0.01784335821866989,
      "rewards/margins_max": 0.025632936507463455,
      "rewards/margins_min": 0.010053779929876328,
      "rewards/margins_std": 0.011016124859452248,
      "rewards/rejected": -0.010165175423026085,
      "step": 220
    },
    {
      "epoch": 0.13,
      "grad_norm": 0.474609375,
      "learning_rate": 4.983356165200751e-07,
      "logits/chosen": 0.07358375936746597,
      "logits/rejected": 0.617803692817688,
      "logps/chosen": -276.56536865234375,
      "logps/rejected": -237.3117218017578,
      "loss": 0.6848,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.0074386284686625,
      "rewards/margins": 0.01824963092803955,
      "rewards/margins_max": 0.026552444323897362,
      "rewards/margins_min": 0.00994681753218174,
      "rewards/margins_std": 0.01174195110797882,
      "rewards/rejected": -0.010811002925038338,
      "step": 230
    },
    {
      "epoch": 0.14,
      "grad_norm": 0.4296875,
      "learning_rate": 4.977013697281864e-07,
      "logits/chosen": 0.23069170117378235,
      "logits/rejected": 0.546830952167511,
      "logps/chosen": -229.92764282226562,
      "logps/rejected": -231.63357543945312,
      "loss": 0.6848,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.005361995659768581,
      "rewards/margins": 0.015256190672516823,
      "rewards/margins_max": 0.022752556949853897,
      "rewards/margins_min": 0.007759819272905588,
      "rewards/margins_std": 0.010601467452943325,
      "rewards/rejected": -0.009894194081425667,
      "step": 240
    },
    {
      "epoch": 0.15,
      "grad_norm": 0.412109375,
      "learning_rate": 4.969655004749673e-07,
      "logits/chosen": 0.05646086856722832,
      "logits/rejected": 0.3687281012535095,
      "logps/chosen": -203.8467559814453,
      "logps/rejected": -216.0234375,
      "loss": 0.6846,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.002810864243656397,
      "rewards/margins": 0.014029537327587605,
      "rewards/margins_max": 0.019475888460874557,
      "rewards/margins_min": 0.008583188988268375,
      "rewards/margins_std": 0.007702300790697336,
      "rewards/rejected": -0.011218673549592495,
      "step": 250
    },
    {
      "epoch": 0.15,
      "grad_norm": 0.490234375,
      "learning_rate": 4.961283106596155e-07,
      "logits/chosen": 0.1512751430273056,
      "logits/rejected": 0.5323320627212524,
      "logps/chosen": -256.96673583984375,
      "logps/rejected": -265.65509033203125,
      "loss": 0.6829,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.011281570419669151,
      "rewards/margins": 0.0202823244035244,
      "rewards/margins_max": 0.02979358099400997,
      "rewards/margins_min": 0.010771063156425953,
      "rewards/margins_std": 0.013450953178107738,
      "rewards/rejected": -0.009000752121210098,
      "step": 260
    },
    {
      "epoch": 0.16,
      "grad_norm": 0.447265625,
      "learning_rate": 4.951901437493054e-07,
      "logits/chosen": 0.08749596029520035,
      "logits/rejected": 0.47565847635269165,
      "logps/chosen": -252.97323608398438,
      "logps/rejected": -220.1329803466797,
      "loss": 0.6826,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.005718126427382231,
      "rewards/margins": 0.019988398998975754,
      "rewards/margins_max": 0.025959456339478493,
      "rewards/margins_min": 0.014017338864505291,
      "rewards/margins_std": 0.008444352075457573,
      "rewards/rejected": -0.014270270243287086,
      "step": 270
    },
    {
      "epoch": 0.16,
      "grad_norm": 0.453125,
      "learning_rate": 4.941513846382779e-07,
      "logits/chosen": 0.31170374155044556,
      "logits/rejected": 0.6478020548820496,
      "logps/chosen": -207.89794921875,
      "logps/rejected": -225.51791381835938,
      "loss": 0.6828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.010051739402115345,
      "rewards/margins": 0.019436318427324295,
      "rewards/margins_max": 0.025176430121064186,
      "rewards/margins_min": 0.013696206733584404,
      "rewards/margins_std": 0.008117742836475372,
      "rewards/rejected": -0.009384581819176674,
      "step": 280
    },
    {
      "epoch": 0.17,
      "grad_norm": 0.431640625,
      "learning_rate": 4.930124594899313e-07,
      "logits/chosen": 0.14136287569999695,
      "logits/rejected": 0.5530031323432922,
      "logps/chosen": -244.9897918701172,
      "logps/rejected": -244.90457153320312,
      "loss": 0.6814,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.0166664756834507,
      "rewards/margins": 0.02829556167125702,
      "rewards/margins_max": 0.037106942385435104,
      "rewards/margins_min": 0.019484177231788635,
      "rewards/margins_std": 0.012461178004741669,
      "rewards/rejected": -0.011629085056483746,
      "step": 290
    },
    {
      "epoch": 0.17,
      "grad_norm": 0.494140625,
      "learning_rate": 4.917738355619842e-07,
      "logits/chosen": 0.2040259838104248,
      "logits/rejected": 0.6138412356376648,
      "logps/chosen": -193.21507263183594,
      "logps/rejected": -194.8699188232422,
      "loss": 0.6796,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.012191513553261757,
      "rewards/margins": 0.026244569569826126,
      "rewards/margins_max": 0.036748819053173065,
      "rewards/margins_min": 0.015740320086479187,
      "rewards/margins_std": 0.014855247922241688,
      "rewards/rejected": -0.01405305415391922,
      "step": 300
    },
    {
      "epoch": 0.18,
      "grad_norm": 0.453125,
      "learning_rate": 4.904360210147762e-07,
      "logits/chosen": 0.1507195234298706,
      "logits/rejected": 0.5720406174659729,
      "logps/chosen": -242.0141143798828,
      "logps/rejected": -216.76132202148438,
      "loss": 0.6791,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.010296806693077087,
      "rewards/margins": 0.02473880909383297,
      "rewards/margins_max": 0.036660365760326385,
      "rewards/margins_min": 0.012817250564694405,
      "rewards/margins_std": 0.0168596301227808,
      "rewards/rejected": -0.014442001469433308,
      "step": 310
    },
    {
      "epoch": 0.19,
      "grad_norm": 0.41796875,
      "learning_rate": 4.8899956470279e-07,
      "logits/chosen": -0.03488525375723839,
      "logits/rejected": 0.40159520506858826,
      "logps/chosen": -218.23812866210938,
      "logps/rejected": -190.8876953125,
      "loss": 0.679,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.014135973528027534,
      "rewards/margins": 0.02363484725356102,
      "rewards/margins_max": 0.036806877702474594,
      "rewards/margins_min": 0.010462815873324871,
      "rewards/margins_std": 0.018628064543008804,
      "rewards/rejected": -0.00949887465685606,
      "step": 320
    },
    {
      "epoch": 0.19,
      "grad_norm": 0.4375,
      "learning_rate": 4.874650559494765e-07,
      "logits/chosen": 0.10674601793289185,
      "logits/rejected": 0.5667238831520081,
      "logps/chosen": -242.5848388671875,
      "logps/rejected": -212.60922241210938,
      "loss": 0.6782,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.008991287089884281,
      "rewards/margins": 0.02689727023243904,
      "rewards/margins_max": 0.03854988515377045,
      "rewards/margins_min": 0.015244655311107635,
      "rewards/margins_std": 0.016479285433888435,
      "rewards/rejected": -0.017905984073877335,
      "step": 330
    },
    {
      "epoch": 0.2,
      "grad_norm": 0.357421875,
      "learning_rate": 4.858331243054782e-07,
      "logits/chosen": 0.09378918260335922,
      "logits/rejected": 0.42793530225753784,
      "logps/chosen": -282.80413818359375,
      "logps/rejected": -245.1541748046875,
      "loss": 0.6796,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.004886592272669077,
      "rewards/margins": 0.021504424512386322,
      "rewards/margins_max": 0.03542860597372055,
      "rewards/margins_min": 0.007580241654068232,
      "rewards/margins_std": 0.019691769033670425,
      "rewards/rejected": -0.016617832705378532,
      "step": 340
    },
    {
      "epoch": 0.2,
      "grad_norm": 0.486328125,
      "learning_rate": 4.841044392903481e-07,
      "logits/chosen": 0.1290682703256607,
      "logits/rejected": 0.6047347784042358,
      "logps/chosen": -232.40908813476562,
      "logps/rejected": -181.57228088378906,
      "loss": 0.6783,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.008800150826573372,
      "rewards/margins": 0.028118547052145004,
      "rewards/margins_max": 0.04057111591100693,
      "rewards/margins_min": 0.015665989369153976,
      "rewards/margins_std": 0.0176105834543705,
      "rewards/rejected": -0.01931839995086193,
      "step": 350
    },
    {
      "epoch": 0.21,
      "grad_norm": 0.435546875,
      "learning_rate": 4.822797101178718e-07,
      "logits/chosen": -0.10504484176635742,
      "logits/rejected": 0.437595933675766,
      "logps/chosen": -256.3827209472656,
      "logps/rejected": -231.28836059570312,
      "loss": 0.6777,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.014989467337727547,
      "rewards/margins": 0.03444572165608406,
      "rewards/margins_max": 0.04873298108577728,
      "rewards/margins_min": 0.02015846036374569,
      "rewards/margins_std": 0.020205235108733177,
      "rewards/rejected": -0.019456254318356514,
      "step": 360
    },
    {
      "epoch": 0.21,
      "grad_norm": 0.390625,
      "learning_rate": 4.803596854051038e-07,
      "logits/chosen": -0.0018104672199115157,
      "logits/rejected": 0.5270112752914429,
      "logps/chosen": -251.33740234375,
      "logps/rejected": -203.73886108398438,
      "loss": 0.6749,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.010898159816861153,
      "rewards/margins": 0.02897489070892334,
      "rewards/margins_max": 0.041702691465616226,
      "rewards/margins_min": 0.016247089952230453,
      "rewards/margins_std": 0.01799982599914074,
      "rewards/rejected": -0.018076732754707336,
      "step": 370
    },
    {
      "epoch": 0.22,
      "grad_norm": 0.3671875,
      "learning_rate": 4.783451528652382e-07,
      "logits/chosen": 0.03281222656369209,
      "logits/rejected": 0.3939230740070343,
      "logps/chosen": -203.0167694091797,
      "logps/rejected": -197.302490234375,
      "loss": 0.6775,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.01019463874399662,
      "rewards/margins": 0.030594149604439735,
      "rewards/margins_max": 0.041967082768678665,
      "rewards/margins_min": 0.019221220165491104,
      "rewards/margins_std": 0.01608375459909439,
      "rewards/rejected": -0.020399510860443115,
      "step": 380
    },
    {
      "epoch": 0.23,
      "grad_norm": 0.4140625,
      "learning_rate": 4.7623693898443963e-07,
      "logits/chosen": 0.06993720680475235,
      "logits/rejected": 0.44206172227859497,
      "logps/chosen": -185.37237548828125,
      "logps/rejected": -187.4385986328125,
      "loss": 0.6751,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.009011445567011833,
      "rewards/margins": 0.03231946378946304,
      "rewards/margins_max": 0.04668620228767395,
      "rewards/margins_min": 0.017952727153897285,
      "rewards/margins_std": 0.02031763456761837,
      "rewards/rejected": -0.02330802008509636,
      "step": 390
    },
    {
      "epoch": 0.23,
      "grad_norm": 0.44140625,
      "learning_rate": 4.740359086827685e-07,
      "logits/chosen": -0.0161175187677145,
      "logits/rejected": 0.4163980484008789,
      "logps/chosen": -239.71432495117188,
      "logps/rejected": -241.2501678466797,
      "loss": 0.6737,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.018473349511623383,
      "rewards/margins": 0.04534245282411575,
      "rewards/margins_max": 0.06162145733833313,
      "rewards/margins_min": 0.02906343713402748,
      "rewards/margins_std": 0.0230219978839159,
      "rewards/rejected": -0.026869099587202072,
      "step": 400
    },
    {
      "epoch": 0.24,
      "grad_norm": 0.359375,
      "learning_rate": 4.7174296495933593e-07,
      "logits/chosen": -0.04076371714472771,
      "logits/rejected": 0.20715077221393585,
      "logps/chosen": -188.3863525390625,
      "logps/rejected": -203.01266479492188,
      "loss": 0.6749,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.011351143009960651,
      "rewards/margins": 0.03776105120778084,
      "rewards/margins_max": 0.05341630056500435,
      "rewards/margins_min": 0.022105801850557327,
      "rewards/margins_std": 0.022139865905046463,
      "rewards/rejected": -0.026409905403852463,
      "step": 410
    },
    {
      "epoch": 0.24,
      "grad_norm": 0.478515625,
      "learning_rate": 4.6935904852183805e-07,
      "logits/chosen": 0.29291218519210815,
      "logits/rejected": 0.5505505800247192,
      "logps/chosen": -203.9456024169922,
      "logps/rejected": -217.8910369873047,
      "loss": 0.6712,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.012085825204849243,
      "rewards/margins": 0.038635291159152985,
      "rewards/margins_max": 0.059398896992206573,
      "rewards/margins_min": 0.017871689051389694,
      "rewards/margins_std": 0.029364168643951416,
      "rewards/rejected": -0.02654946781694889,
      "step": 420
    },
    {
      "epoch": 0.25,
      "grad_norm": 0.431640625,
      "learning_rate": 4.6688513740061965e-07,
      "logits/chosen": 0.12483358383178711,
      "logits/rejected": 0.46587473154067993,
      "logps/chosen": -264.0867004394531,
      "logps/rejected": -292.27685546875,
      "loss": 0.6731,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.019537176936864853,
      "rewards/margins": 0.040542975068092346,
      "rewards/margins_max": 0.05839340761303902,
      "rewards/margins_min": 0.022692536935210228,
      "rewards/margins_std": 0.02524433098733425,
      "rewards/rejected": -0.021005798131227493,
      "step": 430
    },
    {
      "epoch": 0.26,
      "grad_norm": 0.4296875,
      "learning_rate": 4.6432224654742475e-07,
      "logits/chosen": -0.0027520388830453157,
      "logits/rejected": 0.48325324058532715,
      "logps/chosen": -231.2857208251953,
      "logps/rejected": -221.3975372314453,
      "loss": 0.6719,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.017787110060453415,
      "rewards/margins": 0.04569714143872261,
      "rewards/margins_max": 0.06507585942745209,
      "rewards/margins_min": 0.026318421587347984,
      "rewards/margins_std": 0.027405640110373497,
      "rewards/rejected": -0.027910029515624046,
      "step": 440
    },
    {
      "epoch": 0.26,
      "grad_norm": 0.4375,
      "learning_rate": 4.616714274190011e-07,
      "logits/chosen": 0.3332589566707611,
      "logits/rejected": 0.5584608316421509,
      "logps/chosen": -211.74325561523438,
      "logps/rejected": -225.31689453125,
      "loss": 0.6705,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.010198825970292091,
      "rewards/margins": 0.04217001795768738,
      "rewards/margins_max": 0.0582113042473793,
      "rewards/margins_min": 0.026128727942705154,
      "rewards/margins_std": 0.022685810923576355,
      "rewards/rejected": -0.031971193850040436,
      "step": 450
    },
    {
      "epoch": 0.27,
      "grad_norm": 0.435546875,
      "learning_rate": 4.589337675457273e-07,
      "logits/chosen": 0.10014849901199341,
      "logits/rejected": 0.564907431602478,
      "logps/chosen": -217.19985961914062,
      "logps/rejected": -214.29440307617188,
      "loss": 0.6713,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.018607165664434433,
      "rewards/margins": 0.05433148890733719,
      "rewards/margins_max": 0.07488565146923065,
      "rewards/margins_min": 0.033777330070734024,
      "rewards/margins_std": 0.02906796894967556,
      "rewards/rejected": -0.03572431951761246,
      "step": 460
    },
    {
      "epoch": 0.27,
      "grad_norm": 0.4609375,
      "learning_rate": 4.5611039008544007e-07,
      "logits/chosen": 0.13153567910194397,
      "logits/rejected": 0.652635931968689,
      "logps/chosen": -261.8456726074219,
      "logps/rejected": -231.66531372070312,
      "loss": 0.671,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.013766567222774029,
      "rewards/margins": 0.04572372883558273,
      "rewards/margins_max": 0.06320376694202423,
      "rewards/margins_min": 0.028243690729141235,
      "rewards/margins_std": 0.024720508605241776,
      "rewards/rejected": -0.03195716068148613,
      "step": 470
    },
    {
      "epoch": 0.28,
      "grad_norm": 0.419921875,
      "learning_rate": 4.532024533626457e-07,
      "logits/chosen": 0.0050893365405499935,
      "logits/rejected": 0.3075583577156067,
      "logps/chosen": -214.87033081054688,
      "logps/rejected": -231.591064453125,
      "loss": 0.6694,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.012458743527531624,
      "rewards/margins": 0.046287618577480316,
      "rewards/margins_max": 0.06574501842260361,
      "rewards/margins_min": 0.026830215007066727,
      "rewards/margins_std": 0.02751692570745945,
      "rewards/rejected": -0.03382887691259384,
      "step": 480
    },
    {
      "epoch": 0.28,
      "grad_norm": 0.435546875,
      "learning_rate": 4.502111503933032e-07,
      "logits/chosen": 0.16573339700698853,
      "logits/rejected": 0.5059231519699097,
      "logps/chosen": -214.00900268554688,
      "logps/rejected": -226.75070190429688,
      "loss": 0.6705,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.011546745896339417,
      "rewards/margins": 0.03893359750509262,
      "rewards/margins_max": 0.0571872778236866,
      "rewards/margins_min": 0.020679913461208344,
      "rewards/margins_std": 0.0258146021515131,
      "rewards/rejected": -0.027386849746108055,
      "step": 490
    },
    {
      "epoch": 0.29,
      "grad_norm": 0.42578125,
      "learning_rate": 4.471377083953753e-07,
      "logits/chosen": 0.19767063856124878,
      "logits/rejected": 0.6161295175552368,
      "logps/chosen": -211.5915985107422,
      "logps/rejected": -231.336669921875,
      "loss": 0.6672,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.021602794528007507,
      "rewards/margins": 0.05690021067857742,
      "rewards/margins_max": 0.08022460341453552,
      "rewards/margins_min": 0.03357581049203873,
      "rewards/margins_std": 0.032985687255859375,
      "rewards/rejected": -0.03529741242527962,
      "step": 500
    },
    {
      "epoch": 0.3,
      "grad_norm": 0.4609375,
      "learning_rate": 4.4398338828534766e-07,
      "logits/chosen": 0.051334965974092484,
      "logits/rejected": 0.5114815831184387,
      "logps/chosen": -252.36349487304688,
      "logps/rejected": -253.6934051513672,
      "loss": 0.67,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.021400339901447296,
      "rewards/margins": 0.05237139016389847,
      "rewards/margins_max": 0.07569600641727448,
      "rewards/margins_min": 0.029046764597296715,
      "rewards/margins_std": 0.03298599272966385,
      "rewards/rejected": -0.030971046537160873,
      "step": 510
    },
    {
      "epoch": 0.3,
      "grad_norm": 0.40234375,
      "learning_rate": 4.407494841609224e-07,
      "logits/chosen": 0.16097505390644073,
      "logits/rejected": 0.503351092338562,
      "logps/chosen": -187.7499542236328,
      "logps/rejected": -182.64669799804688,
      "loss": 0.6691,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.015485493466258049,
      "rewards/margins": 0.039487432688474655,
      "rewards/margins_max": 0.0597788468003273,
      "rewards/margins_min": 0.019196024164557457,
      "rewards/margins_std": 0.028696388006210327,
      "rewards/rejected": -0.024001937359571457,
      "step": 520
    },
    {
      "epoch": 0.31,
      "grad_norm": 0.462890625,
      "learning_rate": 4.374373227700993e-07,
      "logits/chosen": 0.03560265153646469,
      "logits/rejected": 0.5799299478530884,
      "logps/chosen": -273.8843688964844,
      "logps/rejected": -234.033935546875,
      "loss": 0.6673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.007162511348724365,
      "rewards/margins": 0.0483052022755146,
      "rewards/margins_max": 0.06804867088794708,
      "rewards/margins_min": 0.028561726212501526,
      "rewards/margins_std": 0.027921488508582115,
      "rewards/rejected": -0.04114269092679024,
      "step": 530
    },
    {
      "epoch": 0.31,
      "grad_norm": 0.408203125,
      "learning_rate": 4.340482629668615e-07,
      "logits/chosen": 0.027306120842695236,
      "logits/rejected": 0.671806812286377,
      "logps/chosen": -259.85015869140625,
      "logps/rejected": -201.55807495117188,
      "loss": 0.6673,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.02854643389582634,
      "rewards/margins": 0.0538957342505455,
      "rewards/margins_max": 0.0864059180021286,
      "rewards/margins_min": 0.0213855542242527,
      "rewards/margins_std": 0.045976340770721436,
      "rewards/rejected": -0.025349300354719162,
      "step": 540
    },
    {
      "epoch": 0.32,
      "grad_norm": 0.3515625,
      "learning_rate": 4.30583695153689e-07,
      "logits/chosen": 0.04380347207188606,
      "logits/rejected": 0.4509994089603424,
      "logps/chosen": -273.69775390625,
      "logps/rejected": -259.96966552734375,
      "loss": 0.6693,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.022089816629886627,
      "rewards/margins": 0.056071024388074875,
      "rewards/margins_max": 0.08100839704275131,
      "rewards/margins_min": 0.031133651733398438,
      "rewards/margins_std": 0.035266775637865067,
      "rewards/rejected": -0.033981211483478546,
      "step": 550
    },
    {
      "epoch": 0.32,
      "grad_norm": 0.4140625,
      "learning_rate": 4.2704504071112986e-07,
      "logits/chosen": 0.10579466819763184,
      "logits/rejected": 0.5407041311264038,
      "logps/chosen": -240.98483276367188,
      "logps/rejected": -211.9040985107422,
      "loss": 0.6687,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.017832906916737556,
      "rewards/margins": 0.05916459485888481,
      "rewards/margins_max": 0.08200596272945404,
      "rewards/margins_min": 0.036323241889476776,
      "rewards/margins_std": 0.03230256214737892,
      "rewards/rejected": -0.041331697255373,
      "step": 560
    },
    {
      "epoch": 0.33,
      "grad_norm": 0.376953125,
      "learning_rate": 4.234337514146612e-07,
      "logits/chosen": 0.11410923302173615,
      "logits/rejected": 0.6912606954574585,
      "logps/chosen": -251.16793823242188,
      "logps/rejected": -229.26553344726562,
      "loss": 0.6663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.019808156415820122,
      "rewards/margins": 0.05665863677859306,
      "rewards/margins_max": 0.08191566169261932,
      "rewards/margins_min": 0.0314016118645668,
      "rewards/margins_std": 0.03571882098913193,
      "rewards/rejected": -0.036850474774837494,
      "step": 570
    },
    {
      "epoch": 0.34,
      "grad_norm": 0.357421875,
      "learning_rate": 4.197513088390813e-07,
      "logits/chosen": -0.013543277978897095,
      "logits/rejected": 0.37492939829826355,
      "logps/chosen": -232.13333129882812,
      "logps/rejected": -223.6721954345703,
      "loss": 0.6657,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.014923980459570885,
      "rewards/margins": 0.05013802647590637,
      "rewards/margins_max": 0.07493571937084198,
      "rewards/margins_min": 0.025340333580970764,
      "rewards/margins_std": 0.03506923094391823,
      "rewards/rejected": -0.03521404415369034,
      "step": 580
    },
    {
      "epoch": 0.34,
      "grad_norm": 0.51171875,
      "learning_rate": 4.1599922375067554e-07,
      "logits/chosen": -0.03167729452252388,
      "logits/rejected": 0.535004734992981,
      "logps/chosen": -325.4375915527344,
      "logps/rejected": -253.494873046875,
      "loss": 0.6668,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.01660420373082161,
      "rewards/margins": 0.059089016169309616,
      "rewards/margins_max": 0.08827444911003113,
      "rewards/margins_min": 0.029903585091233253,
      "rewards/margins_std": 0.041274432092905045,
      "rewards/rejected": -0.04248481243848801,
      "step": 590
    },
    {
      "epoch": 0.35,
      "grad_norm": 0.380859375,
      "learning_rate": 4.121790354874065e-07,
      "logits/chosen": 0.05303360894322395,
      "logits/rejected": 0.40770038962364197,
      "logps/chosen": -202.06549072265625,
      "logps/rejected": -214.628173828125,
      "loss": 0.6649,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.005082354880869389,
      "rewards/margins": 0.05396551638841629,
      "rewards/margins_max": 0.07737747579813004,
      "rewards/margins_min": 0.03055354580283165,
      "rewards/margins_std": 0.03310951590538025,
      "rewards/rejected": -0.04888315126299858,
      "step": 600
    },
    {
      "epoch": 0.35,
      "grad_norm": 0.369140625,
      "learning_rate": 4.082923113273822e-07,
      "logits/chosen": 0.11870566755533218,
      "logits/rejected": 0.464911550283432,
      "logps/chosen": -231.35336303710938,
      "logps/rejected": -234.9374237060547,
      "loss": 0.6666,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.01106190960854292,
      "rewards/margins": 0.0625653862953186,
      "rewards/margins_max": 0.08917935192584991,
      "rewards/margins_min": 0.03595142811536789,
      "rewards/margins_std": 0.037637822329998016,
      "rewards/rejected": -0.05150347948074341,
      "step": 610
    },
    {
      "epoch": 0.36,
      "grad_norm": 0.443359375,
      "learning_rate": 4.043406458458609e-07,
      "logits/chosen": 0.09034819900989532,
      "logits/rejected": 0.5873952507972717,
      "logps/chosen": -265.25396728515625,
      "logps/rejected": -214.2862548828125,
      "loss": 0.6628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0020419310312718153,
      "rewards/margins": 0.06574475765228271,
      "rewards/margins_max": 0.08710642158985138,
      "rewards/margins_min": 0.04438310116529465,
      "rewards/margins_std": 0.030209947377443314,
      "rewards/rejected": -0.06370283663272858,
      "step": 620
    },
    {
      "epoch": 0.37,
      "grad_norm": 0.4921875,
      "learning_rate": 4.0032566026105806e-07,
      "logits/chosen": 0.008516276255249977,
      "logits/rejected": 0.6535265445709229,
      "logps/chosen": -260.87298583984375,
      "logps/rejected": -267.5401916503906,
      "loss": 0.663,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.03661227226257324,
      "rewards/margins": 0.07144369184970856,
      "rewards/margins_max": 0.09834811091423035,
      "rewards/margins_min": 0.044539276510477066,
      "rewards/margins_std": 0.03804859146475792,
      "rewards/rejected": -0.03483142331242561,
      "step": 630
    },
    {
      "epoch": 0.37,
      "grad_norm": 0.474609375,
      "learning_rate": 3.9624900176902184e-07,
      "logits/chosen": 0.013054514303803444,
      "logits/rejected": 0.3652392029762268,
      "logps/chosen": -235.1199493408203,
      "logps/rejected": -248.31411743164062,
      "loss": 0.6656,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.014549237675964832,
      "rewards/margins": 0.05561714246869087,
      "rewards/margins_max": 0.08446307480335236,
      "rewards/margins_min": 0.026771211996674538,
      "rewards/margins_std": 0.040794309228658676,
      "rewards/rejected": -0.041067905724048615,
      "step": 640
    },
    {
      "epoch": 0.38,
      "grad_norm": 0.41015625,
      "learning_rate": 3.921123428678511e-07,
      "logits/chosen": 0.022506317123770714,
      "logits/rejected": 0.6284270882606506,
      "logps/chosen": -305.97674560546875,
      "logps/rejected": -239.0786590576172,
      "loss": 0.666,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.020474497228860855,
      "rewards/margins": 0.06788565218448639,
      "rewards/margins_max": 0.09115969389677048,
      "rewards/margins_min": 0.044611603021621704,
      "rewards/margins_std": 0.03291446715593338,
      "rewards/rejected": -0.047411151230335236,
      "step": 650
    },
    {
      "epoch": 0.38,
      "grad_norm": 0.478515625,
      "learning_rate": 3.8791738067153314e-07,
      "logits/chosen": 0.07077694684267044,
      "logits/rejected": 0.5682755708694458,
      "logps/chosen": -231.22695922851562,
      "logps/rejected": -227.6490478515625,
      "loss": 0.6622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03146480768918991,
      "rewards/margins": 0.06544210761785507,
      "rewards/margins_max": 0.0967545360326767,
      "rewards/margins_min": 0.034129686653614044,
      "rewards/margins_std": 0.044282447546720505,
      "rewards/rejected": -0.03397729992866516,
      "step": 660
    },
    {
      "epoch": 0.39,
      "grad_norm": 0.41796875,
      "learning_rate": 3.83665836213682e-07,
      "logits/chosen": 0.12142015993595123,
      "logits/rejected": 0.5390751957893372,
      "logps/chosen": -207.6114501953125,
      "logps/rejected": -215.29849243164062,
      "loss": 0.6636,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.011886438354849815,
      "rewards/margins": 0.05365458130836487,
      "rewards/margins_max": 0.07296213507652283,
      "rewards/margins_min": 0.03434702754020691,
      "rewards/margins_std": 0.027305006980895996,
      "rewards/rejected": -0.0417681448161602,
      "step": 670
    },
    {
      "epoch": 0.39,
      "grad_norm": 0.46875,
      "learning_rate": 3.7935945374146417e-07,
      "logits/chosen": 0.007061509881168604,
      "logits/rejected": 0.3642507493495941,
      "logps/chosen": -236.29788208007812,
      "logps/rejected": -242.33544921875,
      "loss": 0.6631,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.02563950978219509,
      "rewards/margins": 0.05955478549003601,
      "rewards/margins_max": 0.08539506047964096,
      "rewards/margins_min": 0.03371449559926987,
      "rewards/margins_std": 0.036543674767017365,
      "rewards/rejected": -0.03391526639461517,
      "step": 680
    },
    {
      "epoch": 0.4,
      "grad_norm": 0.5234375,
      "learning_rate": 3.75e-07,
      "logits/chosen": 0.08328167349100113,
      "logits/rejected": 0.5527598857879639,
      "logps/chosen": -239.66159057617188,
      "logps/rejected": -235.6712188720703,
      "loss": 0.6622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.023291967809200287,
      "rewards/margins": 0.07459411025047302,
      "rewards/margins_max": 0.1087113469839096,
      "rewards/margins_min": 0.04047687351703644,
      "rewards/margins_std": 0.04824905842542648,
      "rewards/rejected": -0.051302142441272736,
      "step": 690
    },
    {
      "epoch": 0.41,
      "grad_norm": 0.40625,
      "learning_rate": 3.7058926350753517e-07,
      "logits/chosen": 0.04602205008268356,
      "logits/rejected": 0.6276509165763855,
      "logps/chosen": -247.14205932617188,
      "logps/rejected": -208.6519775390625,
      "loss": 0.6614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.022474488243460655,
      "rewards/margins": 0.07001164555549622,
      "rewards/margins_max": 0.09704446792602539,
      "rewards/margins_min": 0.04297882691025734,
      "rewards/margins_std": 0.038230184465646744,
      "rewards/rejected": -0.04753715917468071,
      "step": 700
    },
    {
      "epoch": 0.41,
      "grad_norm": 0.4453125,
      "learning_rate": 3.661290538216798e-07,
      "logits/chosen": 0.291398823261261,
      "logits/rejected": 0.6808168292045593,
      "logps/chosen": -224.65090942382812,
      "logps/rejected": -205.6571807861328,
      "loss": 0.6632,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.0020084187854081392,
      "rewards/margins": 0.05480460077524185,
      "rewards/margins_max": 0.0770978108048439,
      "rewards/margins_min": 0.0325113907456398,
      "rewards/margins_std": 0.031527359038591385,
      "rewards/rejected": -0.05279617756605148,
      "step": 710
    },
    {
      "epoch": 0.42,
      "grad_norm": 0.4375,
      "learning_rate": 3.616212007970159e-07,
      "logits/chosen": 0.05395558476448059,
      "logits/rejected": 0.29135066270828247,
      "logps/chosen": -189.52139282226562,
      "logps/rejected": -215.48080444335938,
      "loss": 0.6633,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.008078034035861492,
      "rewards/margins": 0.05178927257657051,
      "rewards/margins_max": 0.0689278393983841,
      "rewards/margins_min": 0.034650713205337524,
      "rewards/margins_std": 0.024237588047981262,
      "rewards/rejected": -0.043711237609386444,
      "step": 720
    },
    {
      "epoch": 0.42,
      "grad_norm": 0.4609375,
      "learning_rate": 3.5706755383437703e-07,
      "logits/chosen": 0.09721295535564423,
      "logits/rejected": 0.5186147689819336,
      "logps/chosen": -302.69482421875,
      "logps/rejected": -258.5033874511719,
      "loss": 0.6646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.020449183881282806,
      "rewards/margins": 0.052381712943315506,
      "rewards/margins_max": 0.07583948969841003,
      "rewards/margins_min": 0.02892393246293068,
      "rewards/margins_std": 0.0331743024289608,
      "rewards/rejected": -0.0319325253367424,
      "step": 730
    },
    {
      "epoch": 0.43,
      "grad_norm": 0.443359375,
      "learning_rate": 3.5246998112210993e-07,
      "logits/chosen": 0.13969309628009796,
      "logits/rejected": 0.6499422192573547,
      "logps/chosen": -262.07000732421875,
      "logps/rejected": -253.33364868164062,
      "loss": 0.6583,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.020577292889356613,
      "rewards/margins": 0.08194496482610703,
      "rewards/margins_max": 0.10924677550792694,
      "rewards/margins_min": 0.05464313551783562,
      "rewards/margins_std": 0.038610607385635376,
      "rewards/rejected": -0.061367668211460114,
      "step": 740
    },
    {
      "epoch": 0.44,
      "grad_norm": 0.39453125,
      "learning_rate": 3.4783036886962736e-07,
      "logits/chosen": 0.15751202404499054,
      "logits/rejected": 0.583830714225769,
      "logps/chosen": -232.4749298095703,
      "logps/rejected": -251.43881225585938,
      "loss": 0.6642,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.013448268175125122,
      "rewards/margins": 0.06021388620138168,
      "rewards/margins_max": 0.08211688697338104,
      "rewards/margins_min": 0.03831087797880173,
      "rewards/margins_std": 0.030975526198744774,
      "rewards/rejected": -0.04676561802625656,
      "step": 750
    },
    {
      "epoch": 0.44,
      "grad_norm": 0.451171875,
      "learning_rate": 3.4315062053356847e-07,
      "logits/chosen": -0.02616945281624794,
      "logits/rejected": 0.5470731854438782,
      "logps/chosen": -247.7039031982422,
      "logps/rejected": -204.8767547607422,
      "loss": 0.6635,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.02075277827680111,
      "rewards/margins": 0.06478811800479889,
      "rewards/margins_max": 0.09738490730524063,
      "rewards/margins_min": 0.03219131752848625,
      "rewards/margins_std": 0.04609883576631546,
      "rewards/rejected": -0.04403533786535263,
      "step": 760
    },
    {
      "epoch": 0.45,
      "grad_norm": 0.515625,
      "learning_rate": 3.384326560368826e-07,
      "logits/chosen": 0.040539853274822235,
      "logits/rejected": 0.5014762878417969,
      "logps/chosen": -249.2455596923828,
      "logps/rejected": -242.47781372070312,
      "loss": 0.662,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.02245604246854782,
      "rewards/margins": 0.05939044803380966,
      "rewards/margins_max": 0.08405659347772598,
      "rewards/margins_min": 0.03472430631518364,
      "rewards/margins_std": 0.03488319739699364,
      "rewards/rejected": -0.03693440556526184,
      "step": 770
    },
    {
      "epoch": 0.45,
      "grad_norm": 0.5,
      "learning_rate": 3.3367841098115777e-07,
      "logits/chosen": 0.05805939435958862,
      "logits/rejected": 0.47922706604003906,
      "logps/chosen": -286.8292541503906,
      "logps/rejected": -230.5067138671875,
      "loss": 0.6653,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.004244116134941578,
      "rewards/margins": 0.0571456179022789,
      "rewards/margins_max": 0.08360336720943451,
      "rewards/margins_min": 0.030687877908349037,
      "rewards/margins_std": 0.03741690143942833,
      "rewards/rejected": -0.052901506423950195,
      "step": 780
    },
    {
      "epoch": 0.46,
      "grad_norm": 0.40234375,
      "learning_rate": 3.2888983585251713e-07,
      "logits/chosen": 0.11492130905389786,
      "logits/rejected": 0.3956727087497711,
      "logps/chosen": -204.6266632080078,
      "logps/rejected": -208.7443084716797,
      "loss": 0.6606,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.011013984680175781,
      "rewards/margins": 0.057107020169496536,
      "rewards/margins_max": 0.07711775600910187,
      "rewards/margins_min": 0.037096280604600906,
      "rewards/margins_std": 0.02829946205019951,
      "rewards/rejected": -0.046093035489320755,
      "step": 790
    },
    {
      "epoch": 0.46,
      "grad_norm": 0.466796875,
      "learning_rate": 3.240688952214085e-07,
      "logits/chosen": -0.019520867615938187,
      "logits/rejected": 0.34635210037231445,
      "logps/chosen": -278.4693298339844,
      "logps/rejected": -257.54986572265625,
      "loss": 0.6607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.020895112305879593,
      "rewards/margins": 0.08000204712152481,
      "rewards/margins_max": 0.1040647029876709,
      "rewards/margins_min": 0.05593939870595932,
      "rewards/margins_std": 0.034029725939035416,
      "rewards/rejected": -0.05910693481564522,
      "step": 800
    },
    {
      "epoch": 0.47,
      "grad_norm": 0.365234375,
      "learning_rate": 3.192175669366156e-07,
      "logits/chosen": 0.08061734586954117,
      "logits/rejected": 0.440199077129364,
      "logps/chosen": -216.41323852539062,
      "logps/rejected": -240.26333618164062,
      "loss": 0.6611,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.011639273725450039,
      "rewards/margins": 0.061767347157001495,
      "rewards/margins_max": 0.09113974124193192,
      "rewards/margins_min": 0.03239493444561958,
      "rewards/margins_std": 0.04153885692358017,
      "rewards/rejected": -0.050128065049648285,
      "step": 810
    },
    {
      "epoch": 0.48,
      "grad_norm": 0.435546875,
      "learning_rate": 3.14337841313822e-07,
      "logits/chosen": 0.2162504643201828,
      "logits/rejected": 0.6251672506332397,
      "logps/chosen": -249.9015655517578,
      "logps/rejected": -198.54403686523438,
      "loss": 0.6629,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.008589675650000572,
      "rewards/margins": 0.05789928883314133,
      "rewards/margins_max": 0.07874341309070587,
      "rewards/margins_min": 0.03705517202615738,
      "rewards/margins_std": 0.029478034004569054,
      "rewards/rejected": -0.0493096187710762,
      "step": 820
    },
    {
      "epoch": 0.48,
      "grad_norm": 0.443359375,
      "learning_rate": 3.094317203190603e-07,
      "logits/chosen": -0.0029448375571519136,
      "logits/rejected": 0.4555005431175232,
      "logps/chosen": -240.8060760498047,
      "logps/rejected": -222.56246948242188,
      "loss": 0.6561,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.022363275289535522,
      "rewards/margins": 0.08168495446443558,
      "rewards/margins_max": 0.11077789962291718,
      "rewards/margins_min": 0.052591998130083084,
      "rewards/margins_std": 0.04114364832639694,
      "rewards/rejected": -0.059321679174900055,
      "step": 830
    },
    {
      "epoch": 0.49,
      "grad_norm": 0.38671875,
      "learning_rate": 3.045012167473814e-07,
      "logits/chosen": 0.1808149516582489,
      "logits/rejected": 0.5233570337295532,
      "logps/chosen": -263.43255615234375,
      "logps/rejected": -270.8913269042969,
      "loss": 0.6616,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.02442259155213833,
      "rewards/margins": 0.0733276903629303,
      "rewards/margins_max": 0.104800745844841,
      "rewards/margins_min": 0.041854631155729294,
      "rewards/margins_std": 0.04450962692499161,
      "rewards/rejected": -0.04890510439872742,
      "step": 840
    },
    {
      "epoch": 0.49,
      "grad_norm": 0.4140625,
      "learning_rate": 2.995483533970809e-07,
      "logits/chosen": 0.2622363269329071,
      "logits/rejected": 0.7754552960395813,
      "logps/chosen": -228.362060546875,
      "logps/rejected": -187.44383239746094,
      "loss": 0.6618,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.011710538528859615,
      "rewards/margins": 0.06277038902044296,
      "rewards/margins_max": 0.08341649174690247,
      "rewards/margins_min": 0.04212428256869316,
      "rewards/margins_std": 0.029198000207543373,
      "rewards/rejected": -0.05105985328555107,
      "step": 850
    },
    {
      "epoch": 0.5,
      "grad_norm": 0.453125,
      "learning_rate": 2.9457516223982235e-07,
      "logits/chosen": 0.11260411888360977,
      "logits/rejected": 0.47127556800842285,
      "logps/chosen": -251.4638214111328,
      "logps/rejected": -251.6316680908203,
      "loss": 0.6609,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.009782608598470688,
      "rewards/margins": 0.07295442372560501,
      "rewards/margins_max": 0.10423107445240021,
      "rewards/margins_min": 0.04167778044939041,
      "rewards/margins_std": 0.044231854379177094,
      "rewards/rejected": -0.06317181885242462,
      "step": 860
    },
    {
      "epoch": 0.5,
      "grad_norm": 0.44921875,
      "learning_rate": 2.895836835869962e-07,
      "logits/chosen": 0.03560788184404373,
      "logits/rejected": 0.4069921374320984,
      "logps/chosen": -228.38876342773438,
      "logps/rejected": -221.29638671875,
      "loss": 0.662,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.009866083040833473,
      "rewards/margins": 0.06033489108085632,
      "rewards/margins_max": 0.09506522119045258,
      "rewards/margins_min": 0.02560456469655037,
      "rewards/margins_std": 0.0491160973906517,
      "rewards/rejected": -0.050468809902668,
      "step": 870
    },
    {
      "epoch": 0.51,
      "grad_norm": 0.48046875,
      "learning_rate": 2.845759652526574e-07,
      "logits/chosen": 0.07124204933643341,
      "logits/rejected": 0.5192992687225342,
      "logps/chosen": -234.10836791992188,
      "logps/rejected": -189.55230712890625,
      "loss": 0.66,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.01570773683488369,
      "rewards/margins": 0.05234966799616814,
      "rewards/margins_max": 0.07433562725782394,
      "rewards/margins_min": 0.030363699421286583,
      "rewards/margins_std": 0.031092852354049683,
      "rewards/rejected": -0.036641925573349,
      "step": 880
    },
    {
      "epoch": 0.52,
      "grad_norm": 0.427734375,
      "learning_rate": 2.795540617133853e-07,
      "logits/chosen": 0.24306873977184296,
      "logits/rejected": 0.4881308674812317,
      "logps/chosen": -233.5541534423828,
      "logps/rejected": -271.29119873046875,
      "loss": 0.6601,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.0062574222683906555,
      "rewards/margins": 0.06694331020116806,
      "rewards/margins_max": 0.0913429707288742,
      "rewards/margins_min": 0.04254365712404251,
      "rewards/margins_std": 0.03450632840394974,
      "rewards/rejected": -0.060685895383358,
      "step": 890
    },
    {
      "epoch": 0.52,
      "grad_norm": 0.40234375,
      "learning_rate": 2.7452003326540995e-07,
      "logits/chosen": 0.1885126382112503,
      "logits/rejected": 0.6096329689025879,
      "logps/chosen": -223.55380249023438,
      "logps/rejected": -210.834716796875,
      "loss": 0.6613,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.01565275713801384,
      "rewards/margins": 0.0681251734495163,
      "rewards/margins_max": 0.0929432287812233,
      "rewards/margins_min": 0.043307114392519,
      "rewards/margins_std": 0.035098038613796234,
      "rewards/rejected": -0.05247241258621216,
      "step": 900
    },
    {
      "epoch": 0.53,
      "grad_norm": 0.369140625,
      "learning_rate": 2.694759451793508e-07,
      "logits/chosen": 0.3056187033653259,
      "logits/rejected": 0.5238193273544312,
      "logps/chosen": -180.62220764160156,
      "logps/rejected": -202.76705932617188,
      "loss": 0.6628,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.005610722117125988,
      "rewards/margins": 0.053133320063352585,
      "rewards/margins_max": 0.0700041875243187,
      "rewards/margins_min": 0.03626246377825737,
      "rewards/margins_std": 0.023858997970819473,
      "rewards/rejected": -0.04752260446548462,
      "step": 910
    },
    {
      "epoch": 0.53,
      "grad_norm": 0.48828125,
      "learning_rate": 2.644238668529146e-07,
      "logits/chosen": 0.21234102547168732,
      "logits/rejected": 0.48591142892837524,
      "logps/chosen": -223.54971313476562,
      "logps/rejected": -248.9346466064453,
      "loss": 0.6607,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.017756493762135506,
      "rewards/margins": 0.07771660387516022,
      "rewards/margins_max": 0.11433382332324982,
      "rewards/margins_min": 0.04109939560294151,
      "rewards/margins_std": 0.05178455635905266,
      "rewards/rejected": -0.05996011570096016,
      "step": 920
    },
    {
      "epoch": 0.54,
      "grad_norm": 0.396484375,
      "learning_rate": 2.593658709619001e-07,
      "logits/chosen": 0.11299429088830948,
      "logits/rejected": 0.5906545519828796,
      "logps/chosen": -222.49609375,
      "logps/rejected": -204.37290954589844,
      "loss": 0.6601,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.02080368809401989,
      "rewards/margins": 0.07051359862089157,
      "rewards/margins_max": 0.10480418056249619,
      "rewards/margins_min": 0.03622300922870636,
      "rewards/margins_std": 0.048494212329387665,
      "rewards/rejected": -0.04970990866422653,
      "step": 930
    },
    {
      "epoch": 0.55,
      "grad_norm": 0.423828125,
      "learning_rate": 2.5430403260985807e-07,
      "logits/chosen": 0.11868913471698761,
      "logits/rejected": 0.5508742332458496,
      "logps/chosen": -212.3166961669922,
      "logps/rejected": -219.1356658935547,
      "loss": 0.6583,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.021529385820031166,
      "rewards/margins": 0.06332559883594513,
      "rewards/margins_max": 0.0937047004699707,
      "rewards/margins_min": 0.03294649347662926,
      "rewards/margins_std": 0.042962536215782166,
      "rewards/rejected": -0.04179621487855911,
      "step": 940
    },
    {
      "epoch": 0.55,
      "grad_norm": 0.470703125,
      "learning_rate": 2.4924042847675503e-07,
      "logits/chosen": 0.06126406043767929,
      "logits/rejected": 0.5420705080032349,
      "logps/chosen": -294.85845947265625,
      "logps/rejected": -215.2727813720703,
      "loss": 0.661,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.007373870350420475,
      "rewards/margins": 0.05419896915555,
      "rewards/margins_max": 0.08067617565393448,
      "rewards/margins_min": 0.02772175334393978,
      "rewards/margins_std": 0.03744443506002426,
      "rewards/rejected": -0.0468250997364521,
      "step": 950
    },
    {
      "epoch": 0.56,
      "grad_norm": 0.47265625,
      "learning_rate": 2.441771359669902e-07,
      "logits/chosen": 0.13893456757068634,
      "logits/rejected": 0.4921324849128723,
      "logps/chosen": -235.5193634033203,
      "logps/rejected": -225.794189453125,
      "loss": 0.6607,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.012106789276003838,
      "rewards/margins": 0.06842382997274399,
      "rewards/margins_max": 0.100715771317482,
      "rewards/margins_min": 0.03613189607858658,
      "rewards/margins_std": 0.045667704194784164,
      "rewards/rejected": -0.056317038834095,
      "step": 960
    },
    {
      "epoch": 0.56,
      "grad_norm": 0.443359375,
      "learning_rate": 2.391162323571161e-07,
      "logits/chosen": 0.07089251279830933,
      "logits/rejected": 0.48170119524002075,
      "logps/chosen": -230.9342498779297,
      "logps/rejected": -226.3340301513672,
      "loss": 0.6617,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.010878843255341053,
      "rewards/margins": 0.06217268109321594,
      "rewards/margins_max": 0.08883620798587799,
      "rewards/margins_min": 0.03550915792584419,
      "rewards/margins_std": 0.037707917392253876,
      "rewards/rejected": -0.051293838769197464,
      "step": 970
    },
    {
      "epoch": 0.57,
      "grad_norm": 0.42578125,
      "learning_rate": 2.340597939436097e-07,
      "logits/chosen": 0.03681742399930954,
      "logits/rejected": 0.5955736041069031,
      "logps/chosen": -234.0045166015625,
      "logps/rejected": -216.2124786376953,
      "loss": 0.6614,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.0253006462007761,
      "rewards/margins": 0.06550078094005585,
      "rewards/margins_max": 0.0953935831785202,
      "rewards/margins_min": 0.035607993602752686,
      "rewards/margins_std": 0.04227479174733162,
      "rewards/rejected": -0.0402001328766346,
      "step": 980
    },
    {
      "epoch": 0.57,
      "grad_norm": 0.42578125,
      "learning_rate": 2.2900989519104796e-07,
      "logits/chosen": 0.1664225161075592,
      "logits/rejected": 0.4196982979774475,
      "logps/chosen": -182.28829956054688,
      "logps/rejected": -211.08865356445312,
      "loss": 0.6625,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.0058049350045621395,
      "rewards/margins": 0.06564933061599731,
      "rewards/margins_max": 0.09529349207878113,
      "rewards/margins_min": 0.036005161702632904,
      "rewards/margins_std": 0.04192318022251129,
      "rewards/rejected": -0.05984439328312874,
      "step": 990
    },
    {
      "epoch": 0.58,
      "grad_norm": 0.4375,
      "learning_rate": 2.2396860788103353e-07,
      "logits/chosen": -0.04069889336824417,
      "logits/rejected": 0.4455093741416931,
      "logps/chosen": -208.73477172851562,
      "logps/rejected": -199.85501098632812,
      "loss": 0.6608,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.015201890841126442,
      "rewards/margins": 0.08097913861274719,
      "rewards/margins_max": 0.11325138807296753,
      "rewards/margins_min": 0.04870688170194626,
      "rewards/margins_std": 0.04563985764980316,
      "rewards/rejected": -0.0657772421836853,
      "step": 1000
    },
    {
      "epoch": 0.59,
      "grad_norm": 0.451171875,
      "learning_rate": 2.1893800026222083e-07,
      "logits/chosen": 0.24370861053466797,
      "logits/rejected": 0.655241847038269,
      "logps/chosen": -239.9451446533203,
      "logps/rejected": -255.0171356201172,
      "loss": 0.6612,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.01818387396633625,
      "rewards/margins": 0.06645138561725616,
      "rewards/margins_max": 0.0944729745388031,
      "rewards/margins_min": 0.03842979669570923,
      "rewards/margins_std": 0.039628516882658005,
      "rewards/rejected": -0.048267509788274765,
      "step": 1010
    },
    {
      "epoch": 0.59,
      "grad_norm": 0.376953125,
      "learning_rate": 2.1392013620179336e-07,
      "logits/chosen": -0.15726599097251892,
      "logits/rejected": 0.27727076411247253,
      "logps/chosen": -208.62881469726562,
      "logps/rejected": -205.62429809570312,
      "loss": 0.6593,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.012712801806628704,
      "rewards/margins": 0.07130307704210281,
      "rewards/margins_max": 0.09740529954433441,
      "rewards/margins_min": 0.04520086199045181,
      "rewards/margins_std": 0.03691411018371582,
      "rewards/rejected": -0.05859028175473213,
      "step": 1020
    },
    {
      "epoch": 0.6,
      "grad_norm": 0.373046875,
      "learning_rate": 2.0891707433873623e-07,
      "logits/chosen": 0.2577076256275177,
      "logits/rejected": 0.5587279796600342,
      "logps/chosen": -232.6507568359375,
      "logps/rejected": -236.791015625,
      "loss": 0.6608,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.007417677901685238,
      "rewards/margins": 0.06323407590389252,
      "rewards/margins_max": 0.09169165790081024,
      "rewards/margins_min": 0.03477650135755539,
      "rewards/margins_std": 0.040245089679956436,
      "rewards/rejected": -0.055816400796175,
      "step": 1030
    },
    {
      "epoch": 0.6,
      "grad_norm": 0.4609375,
      "learning_rate": 2.039308672392556e-07,
      "logits/chosen": 0.09692186862230301,
      "logits/rejected": 0.5365327000617981,
      "logps/chosen": -220.7172393798828,
      "logps/rejected": -204.85055541992188,
      "loss": 0.6567,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.016125962138175964,
      "rewards/margins": 0.06824339926242828,
      "rewards/margins_max": 0.10508973896503448,
      "rewards/margins_min": 0.03139704838395119,
      "rewards/margins_std": 0.052108604460954666,
      "rewards/rejected": -0.05211742967367172,
      "step": 1040
    },
    {
      "epoch": 0.61,
      "grad_norm": 0.36328125,
      "learning_rate": 1.9896356055468845e-07,
      "logits/chosen": 0.24312233924865723,
      "logits/rejected": 0.5007752180099487,
      "logps/chosen": -217.9171600341797,
      "logps/rejected": -255.72866821289062,
      "loss": 0.6605,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.015429767780005932,
      "rewards/margins": 0.06471355259418488,
      "rewards/margins_max": 0.09141434729099274,
      "rewards/margins_min": 0.03801275044679642,
      "rewards/margins_std": 0.03776064142584801,
      "rewards/rejected": -0.04928378015756607,
      "step": 1050
    },
    {
      "epoch": 0.61,
      "grad_norm": 0.359375,
      "learning_rate": 1.940171921822496e-07,
      "logits/chosen": 0.007707296404987574,
      "logits/rejected": 0.3314017653465271,
      "logps/chosen": -218.86654663085938,
      "logps/rejected": -214.7074737548828,
      "loss": 0.6625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.010595353320240974,
      "rewards/margins": 0.05604109913110733,
      "rewards/margins_max": 0.08353577554225922,
      "rewards/margins_min": 0.028546428307890892,
      "rewards/margins_std": 0.03888333961367607,
      "rewards/rejected": -0.045445747673511505,
      "step": 1060
    },
    {
      "epoch": 0.62,
      "grad_norm": 0.421875,
      "learning_rate": 1.8909379142895977e-07,
      "logits/chosen": 0.08975931257009506,
      "logits/rejected": 0.49662691354751587,
      "logps/chosen": -243.73941040039062,
      "logps/rejected": -218.0565643310547,
      "loss": 0.6628,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.017341626808047295,
      "rewards/margins": 0.06548301875591278,
      "rewards/margins_max": 0.10044316947460175,
      "rewards/margins_min": 0.030522847548127174,
      "rewards/margins_std": 0.0494411401450634,
      "rewards/rejected": -0.04814138263463974,
      "step": 1070
    },
    {
      "epoch": 0.63,
      "grad_norm": 0.419921875,
      "learning_rate": 1.841953781790983e-07,
      "logits/chosen": 0.14877240359783173,
      "logits/rejected": 0.32807669043540955,
      "logps/chosen": -201.35398864746094,
      "logps/rejected": -237.98403930664062,
      "loss": 0.6614,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.011331291869282722,
      "rewards/margins": 0.05169866234064102,
      "rewards/margins_max": 0.08101126551628113,
      "rewards/margins_min": 0.02238604798913002,
      "rewards/margins_std": 0.041454292833805084,
      "rewards/rejected": -0.04036737233400345,
      "step": 1080
    },
    {
      "epoch": 0.63,
      "grad_norm": 0.5234375,
      "learning_rate": 1.793239620655211e-07,
      "logits/chosen": 0.10640072822570801,
      "logits/rejected": 0.5526248812675476,
      "logps/chosen": -198.35403442382812,
      "logps/rejected": -196.8388671875,
      "loss": 0.6604,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.0263301283121109,
      "rewards/margins": 0.07441949844360352,
      "rewards/margins_max": 0.1034015566110611,
      "rewards/margins_min": 0.045437444001436234,
      "rewards/margins_std": 0.040986817330121994,
      "rewards/rejected": -0.04808937385678291,
      "step": 1090
    },
    {
      "epoch": 0.64,
      "grad_norm": 0.390625,
      "learning_rate": 1.744815416451847e-07,
      "logits/chosen": 0.1694943606853485,
      "logits/rejected": 0.6004883050918579,
      "logps/chosen": -255.3223114013672,
      "logps/rejected": -243.01541137695312,
      "loss": 0.6625,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.01719365268945694,
      "rewards/margins": 0.06180461123585701,
      "rewards/margins_max": 0.08655586838722229,
      "rewards/margins_min": 0.03705335780978203,
      "rewards/margins_std": 0.03500355780124664,
      "rewards/rejected": -0.04461096227169037,
      "step": 1100
    },
    {
      "epoch": 0.64,
      "grad_norm": 0.4453125,
      "learning_rate": 1.6967010357921446e-07,
      "logits/chosen": 0.11355743557214737,
      "logits/rejected": 0.4874862730503082,
      "logps/chosen": -210.58767700195312,
      "logps/rejected": -219.46701049804688,
      "loss": 0.6618,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.005143271759152412,
      "rewards/margins": 0.061519283801317215,
      "rewards/margins_max": 0.0864943265914917,
      "rewards/margins_min": 0.036544252187013626,
      "rewards/margins_std": 0.035320036113262177,
      "rewards/rejected": -0.05637601017951965,
      "step": 1110
    },
    {
      "epoch": 0.65,
      "grad_norm": 0.439453125,
      "learning_rate": 1.6489162181785255e-07,
      "logits/chosen": 0.15795719623565674,
      "logits/rejected": 0.5425394773483276,
      "logps/chosen": -245.29562377929688,
      "logps/rejected": -233.9000244140625,
      "loss": 0.6602,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.021811651065945625,
      "rewards/margins": 0.07487231492996216,
      "rewards/margins_max": 0.09871380031108856,
      "rewards/margins_min": 0.051030855625867844,
      "rewards/margins_std": 0.03371693566441536,
      "rewards/rejected": -0.05306067317724228,
      "step": 1120
    },
    {
      "epoch": 0.66,
      "grad_norm": 0.361328125,
      "learning_rate": 1.6014805679062183e-07,
      "logits/chosen": -0.04248831048607826,
      "logits/rejected": 0.36503881216049194,
      "logps/chosen": -204.58383178710938,
      "logps/rejected": -203.0003204345703,
      "loss": 0.6607,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.020199652761220932,
      "rewards/margins": 0.08475508540868759,
      "rewards/margins_max": 0.11757893860340118,
      "rewards/margins_min": 0.051931243389844894,
      "rewards/margins_std": 0.046419933438301086,
      "rewards/rejected": -0.06455543637275696,
      "step": 1130
    },
    {
      "epoch": 0.66,
      "grad_norm": 0.482421875,
      "learning_rate": 1.5544135460203527e-07,
      "logits/chosen": 0.250204861164093,
      "logits/rejected": 0.5448838472366333,
      "logps/chosen": -212.43508911132812,
      "logps/rejected": -247.50747680664062,
      "loss": 0.6601,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.013406927697360516,
      "rewards/margins": 0.07055126130580902,
      "rewards/margins_max": 0.09891954064369202,
      "rewards/margins_min": 0.04218297451734543,
      "rewards/margins_std": 0.04011881351470947,
      "rewards/rejected": -0.05714433267712593,
      "step": 1140
    },
    {
      "epoch": 0.67,
      "grad_norm": 0.408203125,
      "learning_rate": 1.5077344623318388e-07,
      "logits/chosen": 0.08146306127309799,
      "logits/rejected": 0.5028539896011353,
      "logps/chosen": -244.5470733642578,
      "logps/rejected": -203.9750213623047,
      "loss": 0.6622,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.00543981185182929,
      "rewards/margins": 0.0606420524418354,
      "rewards/margins_max": 0.09149619191884995,
      "rewards/margins_min": 0.029787922278046608,
      "rewards/margins_std": 0.043634332716464996,
      "rewards/rejected": -0.05520225316286087,
      "step": 1150
    },
    {
      "epoch": 0.67,
      "grad_norm": 0.4921875,
      "learning_rate": 1.461462467495284e-07,
      "logits/chosen": 0.09238779544830322,
      "logits/rejected": 0.5282326340675354,
      "logps/chosen": -239.08853149414062,
      "logps/rejected": -234.31228637695312,
      "loss": 0.6582,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.020727628841996193,
      "rewards/margins": 0.07139938324689865,
      "rewards/margins_max": 0.0972491055727005,
      "rewards/margins_min": 0.045549679547548294,
      "rewards/margins_std": 0.036557018756866455,
      "rewards/rejected": -0.0506717674434185,
      "step": 1160
    },
    {
      "epoch": 0.68,
      "grad_norm": 0.400390625,
      "learning_rate": 1.4156165451522028e-07,
      "logits/chosen": 0.08472833782434464,
      "logits/rejected": 0.5027869939804077,
      "logps/chosen": -205.4404754638672,
      "logps/rejected": -202.98440551757812,
      "loss": 0.663,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.011948509141802788,
      "rewards/margins": 0.06199117749929428,
      "rewards/margins_max": 0.08956360816955566,
      "rewards/margins_min": 0.03441876173019409,
      "rewards/margins_std": 0.038993291556835175,
      "rewards/rejected": -0.05004267022013664,
      "step": 1170
    },
    {
      "epoch": 0.68,
      "grad_norm": 0.470703125,
      "learning_rate": 1.3702155041427543e-07,
      "logits/chosen": 0.1654224544763565,
      "logits/rejected": 0.39103928208351135,
      "logps/chosen": -221.5464630126953,
      "logps/rejected": -246.1484832763672,
      "loss": 0.6611,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.008782127872109413,
      "rewards/margins": 0.05567712336778641,
      "rewards/margins_max": 0.07324758917093277,
      "rewards/margins_min": 0.038106657564640045,
      "rewards/margins_std": 0.024848390370607376,
      "rewards/rejected": -0.046894993633031845,
      "step": 1180
    },
    {
      "epoch": 0.69,
      "grad_norm": 0.4375,
      "learning_rate": 1.3252779707891902e-07,
      "logits/chosen": 0.009541223756968975,
      "logits/rejected": 0.48217493295669556,
      "logps/chosen": -272.9510192871094,
      "logps/rejected": -204.46435546875,
      "loss": 0.6611,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.009134182706475258,
      "rewards/margins": 0.05944829061627388,
      "rewards/margins_max": 0.08002766221761703,
      "rewards/margins_min": 0.03886892646551132,
      "rewards/margins_std": 0.02910362184047699,
      "rewards/rejected": -0.05031410977244377,
      "step": 1190
    },
    {
      "epoch": 0.7,
      "grad_norm": 0.462890625,
      "learning_rate": 1.2808223812541774e-07,
      "logits/chosen": 0.07254563271999359,
      "logits/rejected": 0.47662535309791565,
      "logps/chosen": -241.54336547851562,
      "logps/rejected": -211.88424682617188,
      "loss": 0.6606,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0020990788470953703,
      "rewards/margins": 0.05149079114198685,
      "rewards/margins_max": 0.08034542202949524,
      "rewards/margins_min": 0.022636160254478455,
      "rewards/margins_std": 0.040806613862514496,
      "rewards/rejected": -0.04939170926809311,
      "step": 1200
    },
    {
      "epoch": 0.7,
      "grad_norm": 0.4375,
      "learning_rate": 1.2368669739771469e-07,
      "logits/chosen": 0.07886068522930145,
      "logits/rejected": 0.4947189390659332,
      "logps/chosen": -206.33993530273438,
      "logps/rejected": -212.7965850830078,
      "loss": 0.6578,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.009903495199978352,
      "rewards/margins": 0.0682389959692955,
      "rewards/margins_max": 0.09637950360774994,
      "rewards/margins_min": 0.04009848088026047,
      "rewards/margins_std": 0.03979669511318207,
      "rewards/rejected": -0.058335501700639725,
      "step": 1210
    },
    {
      "epoch": 0.71,
      "grad_norm": 0.439453125,
      "learning_rate": 1.1934297821917497e-07,
      "logits/chosen": -0.18527595698833466,
      "logits/rejected": 0.35417476296424866,
      "logps/chosen": -271.8248291015625,
      "logps/rejected": -208.87966918945312,
      "loss": 0.6619,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.014687316492199898,
      "rewards/margins": 0.05254317447543144,
      "rewards/margins_max": 0.0765123963356018,
      "rewards/margins_min": 0.028573954477906227,
      "rewards/margins_std": 0.03389759734272957,
      "rewards/rejected": -0.03785586357116699,
      "step": 1220
    },
    {
      "epoch": 0.71,
      "grad_norm": 0.40234375,
      "learning_rate": 1.1505286265275094e-07,
      "logits/chosen": 0.09351782500743866,
      "logits/rejected": 0.5304566621780396,
      "logps/chosen": -217.6367645263672,
      "logps/rejected": -209.18603515625,
      "loss": 0.666,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.01146542839705944,
      "rewards/margins": 0.07028119266033173,
      "rewards/margins_max": 0.10538403689861298,
      "rewards/margins_min": 0.03517835959792137,
      "rewards/margins_std": 0.0496429018676281,
      "rewards/rejected": -0.05881576985120773,
      "step": 1230
    },
    {
      "epoch": 0.72,
      "grad_norm": 0.390625,
      "learning_rate": 1.1081811076986963e-07,
      "logits/chosen": 0.026241421699523926,
      "logits/rejected": 0.6041153073310852,
      "logps/chosen": -228.3728790283203,
      "logps/rejected": -190.1019287109375,
      "loss": 0.6596,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.016418198123574257,
      "rewards/margins": 0.0706411749124527,
      "rewards/margins_max": 0.09941698610782623,
      "rewards/margins_min": 0.041865330189466476,
      "rewards/margins_std": 0.04069516435265541,
      "rewards/rejected": -0.054222963750362396,
      "step": 1240
    },
    {
      "epoch": 0.73,
      "grad_norm": 0.427734375,
      "learning_rate": 1.0664045992834184e-07,
      "logits/chosen": 0.19840288162231445,
      "logits/rejected": 0.5584182143211365,
      "logps/chosen": -254.10147094726562,
      "logps/rejected": -256.0483703613281,
      "loss": 0.6583,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.012557362206280231,
      "rewards/margins": 0.06964166462421417,
      "rewards/margins_max": 0.09085742384195328,
      "rewards/margins_min": 0.04842590540647507,
      "rewards/margins_std": 0.030003610998392105,
      "rewards/rejected": -0.057084303349256516,
      "step": 1250
    },
    {
      "epoch": 0.73,
      "grad_norm": 0.484375,
      "learning_rate": 1.0252162405959042e-07,
      "logits/chosen": -0.029180001467466354,
      "logits/rejected": 0.4648149609565735,
      "logps/chosen": -273.28375244140625,
      "logps/rejected": -244.730712890625,
      "loss": 0.6602,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.02007482200860977,
      "rewards/margins": 0.06700652837753296,
      "rewards/margins_max": 0.10410724580287933,
      "rewards/margins_min": 0.029905814677476883,
      "rewards/margins_std": 0.05246833711862564,
      "rewards/rejected": -0.04693170636892319,
      "step": 1260
    },
    {
      "epoch": 0.74,
      "grad_norm": 0.494140625,
      "learning_rate": 9.846329296548963e-08,
      "logits/chosen": -0.017562460154294968,
      "logits/rejected": 0.4763096868991852,
      "logps/chosen": -269.8515625,
      "logps/rejected": -263.83148193359375,
      "loss": 0.6598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.010733803734183311,
      "rewards/margins": 0.07448114454746246,
      "rewards/margins_max": 0.10118886083364487,
      "rewards/margins_min": 0.04777342826128006,
      "rewards/margins_std": 0.03777041286230087,
      "rewards/rejected": -0.0637473464012146,
      "step": 1270
    },
    {
      "epoch": 0.74,
      "grad_norm": 0.486328125,
      "learning_rate": 9.446713162510341e-08,
      "logits/chosen": 0.22771111130714417,
      "logits/rejected": 0.7621752023696899,
      "logps/chosen": -266.06390380859375,
      "logps/rejected": -250.635498046875,
      "loss": 0.6584,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.030348753556609154,
      "rewards/margins": 0.07343067973852158,
      "rewards/margins_max": 0.10677297413349152,
      "rewards/margins_min": 0.040088407695293427,
      "rewards/margins_std": 0.04715309664607048,
      "rewards/rejected": -0.04308192804455757,
      "step": 1280
    },
    {
      "epoch": 0.75,
      "grad_norm": 0.515625,
      "learning_rate": 9.053477951160737e-08,
      "logits/chosen": 0.015399669297039509,
      "logits/rejected": 0.7483765482902527,
      "logps/chosen": -276.5067443847656,
      "logps/rejected": -227.33761596679688,
      "loss": 0.6579,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.026790842413902283,
      "rewards/margins": 0.08279003202915192,
      "rewards/margins_max": 0.11221597343683243,
      "rewards/margins_min": 0.05336407572031021,
      "rewards/margins_std": 0.04161457344889641,
      "rewards/rejected": -0.05599917098879814,
      "step": 1290
    },
    {
      "epoch": 0.75,
      "grad_norm": 0.396484375,
      "learning_rate": 8.666784991967596e-08,
      "logits/chosen": 0.010845961980521679,
      "logits/rejected": 0.42500224709510803,
      "logps/chosen": -213.1592254638672,
      "logps/rejected": -199.2817840576172,
      "loss": 0.6613,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.014592917636036873,
      "rewards/margins": 0.0668349340558052,
      "rewards/margins_max": 0.09872870147228241,
      "rewards/margins_min": 0.03494114801287651,
      "rewards/margins_std": 0.04510461539030075,
      "rewards/rejected": -0.05224201828241348,
      "step": 1300
    },
    {
      "epoch": 0.76,
      "grad_norm": 0.4921875,
      "learning_rate": 8.286792930360823e-08,
      "logits/chosen": 0.25165149569511414,
      "logits/rejected": 0.6992672681808472,
      "logps/chosen": -217.0974884033203,
      "logps/rejected": -202.47030639648438,
      "loss": 0.6599,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.011730032041668892,
      "rewards/margins": 0.0590001717209816,
      "rewards/margins_max": 0.07914995402097702,
      "rewards/margins_min": 0.03885037824511528,
      "rewards/margins_std": 0.02849610149860382,
      "rewards/rejected": -0.04727013781666756,
      "step": 1310
    },
    {
      "epoch": 0.77,
      "grad_norm": 0.52734375,
      "learning_rate": 7.91365766264665e-08,
      "logits/chosen": 0.20514824986457825,
      "logits/rejected": 0.5356392860412598,
      "logps/chosen": -248.6316680908203,
      "logps/rejected": -240.5338134765625,
      "loss": 0.6591,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.010535435751080513,
      "rewards/margins": 0.06282900273799896,
      "rewards/margins_max": 0.09407368302345276,
      "rewards/margins_min": 0.031584326177835464,
      "rewards/margins_std": 0.04418665170669556,
      "rewards/rejected": -0.052293576300144196,
      "step": 1320
    },
    {
      "epoch": 0.77,
      "grad_norm": 0.455078125,
      "learning_rate": 7.547532272049264e-08,
      "logits/chosen": 0.25605538487434387,
      "logits/rejected": 0.6374403238296509,
      "logps/chosen": -255.80410766601562,
      "logps/rejected": -255.73764038085938,
      "loss": 0.6619,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.013418711721897125,
      "rewards/margins": 0.06125851348042488,
      "rewards/margins_max": 0.08139893412590027,
      "rewards/margins_min": 0.04111810773611069,
      "rewards/margins_std": 0.028482843190431595,
      "rewards/rejected": -0.047839801758527756,
      "step": 1330
    },
    {
      "epoch": 0.78,
      "grad_norm": 0.4140625,
      "learning_rate": 7.188566965906584e-08,
      "logits/chosen": 0.10137088596820831,
      "logits/rejected": 0.5515474081039429,
      "logps/chosen": -271.2210693359375,
      "logps/rejected": -272.3622131347656,
      "loss": 0.6598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.00015007219917606562,
      "rewards/margins": 0.06623668223619461,
      "rewards/margins_max": 0.10004226863384247,
      "rewards/margins_min": 0.03243108466267586,
      "rewards/margins_std": 0.04780833050608635,
      "rewards/rejected": -0.06638675183057785,
      "step": 1340
    },
    {
      "epoch": 0.78,
      "grad_norm": 0.412109375,
      "learning_rate": 6.836909014045924e-08,
      "logits/chosen": 0.005819192621856928,
      "logits/rejected": 0.38501212000846863,
      "logps/chosen": -247.23056030273438,
      "logps/rejected": -238.4652557373047,
      "loss": 0.6607,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.01672416180372238,
      "rewards/margins": 0.07304920256137848,
      "rewards/margins_max": 0.10092739760875702,
      "rewards/margins_min": 0.04517098516225815,
      "rewards/margins_std": 0.039425741881132126,
      "rewards/rejected": -0.0563250370323658,
      "step": 1350
    },
    {
      "epoch": 0.79,
      "grad_norm": 0.4609375,
      "learning_rate": 6.492702688364737e-08,
      "logits/chosen": -0.07613168656826019,
      "logits/rejected": 0.20295462012290955,
      "logps/chosen": -203.92233276367188,
      "logps/rejected": -247.69277954101562,
      "loss": 0.6604,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.014894701540470123,
      "rewards/margins": 0.06641440093517303,
      "rewards/margins_max": 0.09283626079559326,
      "rewards/margins_min": 0.039992526173591614,
      "rewards/margins_std": 0.037366170436143875,
      "rewards/rejected": -0.05151969939470291,
      "step": 1360
    },
    {
      "epoch": 0.79,
      "grad_norm": 0.4375,
      "learning_rate": 6.156089203641373e-08,
      "logits/chosen": -0.014948748052120209,
      "logits/rejected": 0.4398605227470398,
      "logps/chosen": -247.429931640625,
      "logps/rejected": -251.06826782226562,
      "loss": 0.6571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0333079919219017,
      "rewards/margins": 0.08266235888004303,
      "rewards/margins_max": 0.10667815059423447,
      "rewards/margins_min": 0.0586465522646904,
      "rewards/margins_std": 0.03396347165107727,
      "rewards/rejected": -0.04935435950756073,
      "step": 1370
    },
    {
      "epoch": 0.8,
      "grad_norm": 0.427734375,
      "learning_rate": 5.827206659599987e-08,
      "logits/chosen": 0.28106218576431274,
      "logits/rejected": 0.7749143242835999,
      "logps/chosen": -222.03665161132812,
      "logps/rejected": -200.11221313476562,
      "loss": 0.6576,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.017674388363957405,
      "rewards/margins": 0.07599468529224396,
      "rewards/margins_max": 0.11385379731655121,
      "rewards/margins_min": 0.038135576993227005,
      "rewards/margins_std": 0.05354086682200432,
      "rewards/rejected": -0.058320302516222,
      "step": 1380
    },
    {
      "epoch": 0.81,
      "grad_norm": 0.50390625,
      "learning_rate": 5.506189984253501e-08,
      "logits/chosen": 0.16949541866779327,
      "logits/rejected": 0.4548502564430237,
      "logps/chosen": -205.447265625,
      "logps/rejected": -221.4696044921875,
      "loss": 0.6611,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.003050294006243348,
      "rewards/margins": 0.06650832295417786,
      "rewards/margins_max": 0.09234586358070374,
      "rewards/margins_min": 0.040670786052942276,
      "rewards/margins_std": 0.036539800465106964,
      "rewards/rejected": -0.06345803290605545,
      "step": 1390
    },
    {
      "epoch": 0.81,
      "grad_norm": 0.482421875,
      "learning_rate": 5.1931708785477506e-08,
      "logits/chosen": 0.11355874687433243,
      "logits/rejected": 0.6481127738952637,
      "logps/chosen": -216.15432739257812,
      "logps/rejected": -187.30389404296875,
      "loss": 0.6592,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.015445582568645477,
      "rewards/margins": 0.05808136984705925,
      "rewards/margins_max": 0.08922155201435089,
      "rewards/margins_min": 0.026941198855638504,
      "rewards/margins_std": 0.04403885826468468,
      "rewards/rejected": -0.04263579100370407,
      "step": 1400
    },
    {
      "epoch": 0.82,
      "grad_norm": 0.380859375,
      "learning_rate": 4.888277762329582e-08,
      "logits/chosen": 0.11872565746307373,
      "logits/rejected": 0.5771151185035706,
      "logps/chosen": -215.25442504882812,
      "logps/rejected": -214.4876251220703,
      "loss": 0.6619,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.01657172292470932,
      "rewards/margins": 0.06676243245601654,
      "rewards/margins_max": 0.0983147844672203,
      "rewards/margins_min": 0.03521009162068367,
      "rewards/margins_std": 0.04462175816297531,
      "rewards/rejected": -0.05019070953130722,
      "step": 1410
    },
    {
      "epoch": 0.82,
      "grad_norm": 0.439453125,
      "learning_rate": 4.591635721661072e-08,
      "logits/chosen": 0.1136382669210434,
      "logits/rejected": 0.5482941269874573,
      "logps/chosen": -243.9540557861328,
      "logps/rejected": -231.51473999023438,
      "loss": 0.6606,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.01714186929166317,
      "rewards/margins": 0.07303180545568466,
      "rewards/margins_max": 0.10039409250020981,
      "rewards/margins_min": 0.045669522136449814,
      "rewards/margins_std": 0.03869611397385597,
      "rewards/rejected": -0.05588993430137634,
      "step": 1420
    },
    {
      "epoch": 0.83,
      "grad_norm": 0.431640625,
      "learning_rate": 4.3033664575015005e-08,
      "logits/chosen": 0.24127981066703796,
      "logits/rejected": 0.6273223161697388,
      "logps/chosen": -258.4788818359375,
      "logps/rejected": -255.1360321044922,
      "loss": 0.6591,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.0254741869866848,
      "rewards/margins": 0.0617264024913311,
      "rewards/margins_max": 0.08791927993297577,
      "rewards/margins_min": 0.035533517599105835,
      "rewards/margins_std": 0.03704233095049858,
      "rewards/rejected": -0.036252211779356,
      "step": 1430
    },
    {
      "epoch": 0.84,
      "grad_norm": 0.4453125,
      "learning_rate": 4.023588235778019e-08,
      "logits/chosen": 0.048088885843753815,
      "logits/rejected": 0.4085961878299713,
      "logps/chosen": -235.32763671875,
      "logps/rejected": -246.94937133789062,
      "loss": 0.6625,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.017656199634075165,
      "rewards/margins": 0.07100087404251099,
      "rewards/margins_max": 0.09923985600471497,
      "rewards/margins_min": 0.042761895805597305,
      "rewards/margins_std": 0.039935946464538574,
      "rewards/rejected": -0.05334467440843582,
      "step": 1440
    },
    {
      "epoch": 0.84,
      "grad_norm": 0.4609375,
      "learning_rate": 3.752415838865664e-08,
      "logits/chosen": -0.09887398779392242,
      "logits/rejected": 0.5310045480728149,
      "logps/chosen": -245.59951782226562,
      "logps/rejected": -266.8290100097656,
      "loss": 0.6586,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.018602244555950165,
      "rewards/margins": 0.08193326741456985,
      "rewards/margins_max": 0.11139090359210968,
      "rewards/margins_min": 0.05247562378644943,
      "rewards/margins_std": 0.041659384965896606,
      "rewards/rejected": -0.06333102285861969,
      "step": 1450
    },
    {
      "epoch": 0.85,
      "grad_norm": 0.439453125,
      "learning_rate": 3.4899605184965206e-08,
      "logits/chosen": 0.03019891306757927,
      "logits/rejected": 0.44324207305908203,
      "logps/chosen": -225.20443725585938,
      "logps/rejected": -183.06094360351562,
      "loss": 0.6609,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.0028962846845388412,
      "rewards/margins": 0.0560896173119545,
      "rewards/margins_max": 0.07679092139005661,
      "rewards/margins_min": 0.035388313233852386,
      "rewards/margins_std": 0.02927606739103794,
      "rewards/rejected": -0.05319333076477051,
      "step": 1460
    },
    {
      "epoch": 0.85,
      "grad_norm": 0.439453125,
      "learning_rate": 3.23632995011732e-08,
      "logits/chosen": -0.06648756563663483,
      "logits/rejected": 0.29680854082107544,
      "logps/chosen": -226.04983520507812,
      "logps/rejected": -258.3298034667969,
      "loss": 0.6587,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03215508535504341,
      "rewards/margins": 0.08979654312133789,
      "rewards/margins_max": 0.12097585201263428,
      "rewards/margins_min": 0.058617234230041504,
      "rewards/margins_std": 0.044094208627939224,
      "rewards/rejected": -0.057641465216875076,
      "step": 1470
    },
    {
      "epoch": 0.86,
      "grad_norm": 0.455078125,
      "learning_rate": 2.991628188714351e-08,
      "logits/chosen": 0.00623916694894433,
      "logits/rejected": 0.48251962661743164,
      "logps/chosen": -313.39935302734375,
      "logps/rejected": -245.91720581054688,
      "loss": 0.6596,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.004381291568279266,
      "rewards/margins": 0.07124367356300354,
      "rewards/margins_max": 0.09969727694988251,
      "rewards/margins_min": 0.04279007390141487,
      "rewards/margins_std": 0.04023946821689606,
      "rewards/rejected": -0.06686238944530487,
      "step": 1480
    },
    {
      "epoch": 0.86,
      "grad_norm": 0.4375,
      "learning_rate": 2.755955626123596e-08,
      "logits/chosen": 0.12439896166324615,
      "logits/rejected": 0.6011586785316467,
      "logps/chosen": -250.7643585205078,
      "logps/rejected": -217.0757293701172,
      "loss": 0.6624,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.018308712169528008,
      "rewards/margins": 0.05787688493728638,
      "rewards/margins_max": 0.09185748547315598,
      "rewards/margins_min": 0.023896273225545883,
      "rewards/margins_std": 0.04805583506822586,
      "rewards/rejected": -0.03956816717982292,
      "step": 1490
    },
    {
      "epoch": 0.87,
      "grad_norm": 0.42578125,
      "learning_rate": 2.5294089498438225e-08,
      "logits/chosen": 0.024487819522619247,
      "logits/rejected": 0.5533932447433472,
      "logps/chosen": -245.57492065429688,
      "logps/rejected": -220.93258666992188,
      "loss": 0.6584,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.010946778580546379,
      "rewards/margins": 0.06493957340717316,
      "rewards/margins_max": 0.0981217697262764,
      "rewards/margins_min": 0.03175736218690872,
      "rewards/margins_std": 0.046926725655794144,
      "rewards/rejected": -0.05399278551340103,
      "step": 1500
    },
    {
      "epoch": 0.88,
      "grad_norm": 0.48046875,
      "learning_rate": 2.312081103369354e-08,
      "logits/chosen": 0.10629892349243164,
      "logits/rejected": 0.5729449987411499,
      "logps/chosen": -227.0969696044922,
      "logps/rejected": -209.62841796875,
      "loss": 0.659,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.013625606894493103,
      "rewards/margins": 0.05797373503446579,
      "rewards/margins_max": 0.0893624946475029,
      "rewards/margins_min": 0.02658497728407383,
      "rewards/margins_std": 0.04439040273427963,
      "rewards/rejected": -0.04434812813997269,
      "step": 1510
    },
    {
      "epoch": 0.88,
      "grad_norm": 0.48046875,
      "learning_rate": 2.104061248058872e-08,
      "logits/chosen": 0.10214777290821075,
      "logits/rejected": 0.4200982451438904,
      "logps/chosen": -213.7083740234375,
      "logps/rejected": -225.8516845703125,
      "loss": 0.6666,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.018484923988580704,
      "rewards/margins": 0.058260779827833176,
      "rewards/margins_max": 0.08636601269245148,
      "rewards/margins_min": 0.030155545100569725,
      "rewards/margins_std": 0.03974680230021477,
      "rewards/rejected": -0.03977585583925247,
      "step": 1520
    },
    {
      "epoch": 0.89,
      "grad_norm": 0.44921875,
      "learning_rate": 1.9054347265559213e-08,
      "logits/chosen": 0.1583404242992401,
      "logits/rejected": 0.6649370193481445,
      "logps/chosen": -259.9563903808594,
      "logps/rejected": -223.4931640625,
      "loss": 0.6565,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.014935478568077087,
      "rewards/margins": 0.07356850802898407,
      "rewards/margins_max": 0.10868100821971893,
      "rewards/margins_min": 0.0384560152888298,
      "rewards/margins_std": 0.049656566232442856,
      "rewards/rejected": -0.058633022010326385,
      "step": 1530
    },
    {
      "epoch": 0.89,
      "grad_norm": 0.498046875,
      "learning_rate": 1.716283027776061e-08,
      "logits/chosen": 0.2019151747226715,
      "logits/rejected": 0.8282853364944458,
      "logps/chosen": -291.37066650390625,
      "logps/rejected": -222.61831665039062,
      "loss": 0.6634,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.016527246683835983,
      "rewards/margins": 0.07255034148693085,
      "rewards/margins_max": 0.1086968407034874,
      "rewards/margins_min": 0.036403849720954895,
      "rewards/margins_std": 0.05111886188387871,
      "rewards/rejected": -0.05602309852838516,
      "step": 1540
    },
    {
      "epoch": 0.9,
      "grad_norm": 0.4296875,
      "learning_rate": 1.536683753475043e-08,
      "logits/chosen": 0.22870250046253204,
      "logits/rejected": 0.4174967408180237,
      "logps/chosen": -219.11306762695312,
      "logps/rejected": -241.36563110351562,
      "loss": 0.6615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0025456459261476994,
      "rewards/margins": 0.059264473617076874,
      "rewards/margins_max": 0.08250005543231964,
      "rewards/margins_min": 0.036028891801834106,
      "rewards/margins_std": 0.032860077917575836,
      "rewards/rejected": -0.061810124665498734,
      "step": 1550
    },
    {
      "epoch": 0.9,
      "grad_norm": 0.390625,
      "learning_rate": 1.3667105864117873e-08,
      "logits/chosen": 0.21612632274627686,
      "logits/rejected": 0.39824485778808594,
      "logps/chosen": -200.84498596191406,
      "logps/rejected": -228.2679901123047,
      "loss": 0.6605,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.008642548695206642,
      "rewards/margins": 0.0651601254940033,
      "rewards/margins_max": 0.10423406213521957,
      "rewards/margins_min": 0.026086175814270973,
      "rewards/margins_std": 0.05525890737771988,
      "rewards/rejected": -0.0565175786614418,
      "step": 1560
    },
    {
      "epoch": 0.91,
      "grad_norm": 0.41796875,
      "learning_rate": 1.2064332601191163e-08,
      "logits/chosen": -0.04893340915441513,
      "logits/rejected": 0.339263916015625,
      "logps/chosen": -222.4666748046875,
      "logps/rejected": -217.02999877929688,
      "loss": 0.6612,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.0008535057422704995,
      "rewards/margins": 0.05954117700457573,
      "rewards/margins_max": 0.0829622894525528,
      "rewards/margins_min": 0.03612007200717926,
      "rewards/margins_std": 0.03312245011329651,
      "rewards/rejected": -0.06039468199014664,
      "step": 1570
    },
    {
      "epoch": 0.92,
      "grad_norm": 0.39453125,
      "learning_rate": 1.0559175302947476e-08,
      "logits/chosen": 0.012552693486213684,
      "logits/rejected": 0.5173078775405884,
      "logps/chosen": -260.0834045410156,
      "logps/rejected": -247.43447875976562,
      "loss": 0.6595,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.011661765165627003,
      "rewards/margins": 0.06366874277591705,
      "rewards/margins_max": 0.09778660535812378,
      "rewards/margins_min": 0.029550885781645775,
      "rewards/margins_std": 0.04824993759393692,
      "rewards/rejected": -0.052006978541612625,
      "step": 1580
    },
    {
      "epoch": 0.92,
      "grad_norm": 0.349609375,
      "learning_rate": 9.152251478242417e-09,
      "logits/chosen": -0.02594194933772087,
      "logits/rejected": 0.4399421215057373,
      "logps/chosen": -212.4099578857422,
      "logps/rejected": -199.73458862304688,
      "loss": 0.6594,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.007081738207489252,
      "rewards/margins": 0.06215248256921768,
      "rewards/margins_max": 0.08854631334543228,
      "rewards/margins_min": 0.03575865179300308,
      "rewards/margins_std": 0.03732650727033615,
      "rewards/rejected": -0.055070746690034866,
      "step": 1590
    },
    {
      "epoch": 0.93,
      "grad_norm": 0.427734375,
      "learning_rate": 7.844138334469425e-09,
      "logits/chosen": 0.4558231234550476,
      "logits/rejected": 0.8965223431587219,
      "logps/chosen": -201.3118438720703,
      "logps/rejected": -192.5732421875,
      "loss": 0.6628,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.009340132586658001,
      "rewards/margins": 0.0616113655269146,
      "rewards/margins_max": 0.09181926399469376,
      "rewards/margins_min": 0.03140346333384514,
      "rewards/margins_std": 0.04272041842341423,
      "rewards/rejected": -0.05227123573422432,
      "step": 1600
    },
    {
      "epoch": 0.93,
      "grad_norm": 0.37109375,
      "learning_rate": 6.635372540753498e-09,
      "logits/chosen": 0.11258337646722794,
      "logits/rejected": 0.6999211311340332,
      "logps/chosen": -240.33975219726562,
      "logps/rejected": -214.0699920654297,
      "loss": 0.6577,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.027147358283400536,
      "rewards/margins": 0.0817473754286766,
      "rewards/margins_max": 0.12004182487726212,
      "rewards/margins_min": 0.0434529110789299,
      "rewards/margins_std": 0.05415653437376022,
      "rewards/rejected": -0.05460001155734062,
      "step": 1610
    },
    {
      "epoch": 0.94,
      "grad_norm": 0.4609375,
      "learning_rate": 5.526450007776435e-09,
      "logits/chosen": 0.1300087720155716,
      "logits/rejected": 0.5238357782363892,
      "logps/chosen": -292.7140197753906,
      "logps/rejected": -246.2644805908203,
      "loss": 0.6611,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.0037552430294454098,
      "rewards/margins": 0.05609096214175224,
      "rewards/margins_max": 0.07447664439678192,
      "rewards/margins_min": 0.03770528361201286,
      "rewards/margins_std": 0.026001274585723877,
      "rewards/rejected": -0.052335720509290695,
      "step": 1620
    },
    {
      "epoch": 0.95,
      "grad_norm": 0.431640625,
      "learning_rate": 4.517825684323323e-09,
      "logits/chosen": 0.18602465093135834,
      "logits/rejected": 0.5172281861305237,
      "logps/chosen": -223.3422088623047,
      "logps/rejected": -241.034912109375,
      "loss": 0.6596,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.00845097191631794,
      "rewards/margins": 0.06410791724920273,
      "rewards/margins_max": 0.09119440615177155,
      "rewards/margins_min": 0.037021439522504807,
      "rewards/margins_std": 0.03830606862902641,
      "rewards/rejected": -0.05565694719552994,
      "step": 1630
    },
    {
      "epoch": 0.95,
      "grad_norm": 0.474609375,
      "learning_rate": 3.6099133706344044e-09,
      "logits/chosen": 0.13008326292037964,
      "logits/rejected": 0.6074930429458618,
      "logps/chosen": -223.1219940185547,
      "logps/rejected": -207.696044921875,
      "loss": 0.6569,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.02304968610405922,
      "rewards/margins": 0.07380314916372299,
      "rewards/margins_max": 0.09590893238782883,
      "rewards/margins_min": 0.05169736221432686,
      "rewards/margins_std": 0.03126230835914612,
      "rewards/rejected": -0.05075346678495407,
      "step": 1640
    },
    {
      "epoch": 0.96,
      "grad_norm": 0.4921875,
      "learning_rate": 2.8030855486386174e-09,
      "logits/chosen": 0.28828924894332886,
      "logits/rejected": 0.6710017919540405,
      "logps/chosen": -256.94903564453125,
      "logps/rejected": -281.40411376953125,
      "loss": 0.6586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.023295782506465912,
      "rewards/margins": 0.071876659989357,
      "rewards/margins_max": 0.09554243832826614,
      "rewards/margins_min": 0.048210885375738144,
      "rewards/margins_std": 0.03346845880150795,
      "rewards/rejected": -0.04858088120818138,
      "step": 1650
    },
    {
      "epoch": 0.96,
      "grad_norm": 0.515625,
      "learning_rate": 2.097673229138286e-09,
      "logits/chosen": 0.16988131403923035,
      "logits/rejected": 0.47897881269454956,
      "logps/chosen": -224.6415557861328,
      "logps/rejected": -232.2594451904297,
      "loss": 0.6587,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.012618700973689556,
      "rewards/margins": 0.07099349051713943,
      "rewards/margins_max": 0.10776303708553314,
      "rewards/margins_min": 0.03422392159700394,
      "rewards/margins_std": 0.0520000159740448,
      "rewards/rejected": -0.05837478116154671,
      "step": 1660
    },
    {
      "epoch": 0.97,
      "grad_norm": 0.44921875,
      "learning_rate": 1.493965816008136e-09,
      "logits/chosen": -0.009510600939393044,
      "logits/rejected": 0.3807966113090515,
      "logps/chosen": -211.14254760742188,
      "logps/rejected": -236.635498046875,
      "loss": 0.6601,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.00740268686786294,
      "rewards/margins": 0.07398059964179993,
      "rewards/margins_max": 0.10376466810703278,
      "rewards/margins_min": 0.04419652372598648,
      "rewards/margins_std": 0.0421210452914238,
      "rewards/rejected": -0.06657791137695312,
      "step": 1670
    },
    {
      "epoch": 0.97,
      "grad_norm": 0.447265625,
      "learning_rate": 9.922109874636875e-10,
      "logits/chosen": 0.19054090976715088,
      "logits/rejected": 0.557522177696228,
      "logps/chosen": -233.7532501220703,
      "logps/rejected": -239.6273651123047,
      "loss": 0.6579,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.015365364961326122,
      "rewards/margins": 0.08128596842288971,
      "rewards/margins_max": 0.11999186128377914,
      "rewards/margins_min": 0.04258008301258087,
      "rewards/margins_std": 0.05473839119076729,
      "rewards/rejected": -0.06592060625553131,
      "step": 1680
    },
    {
      "epoch": 0.98,
      "grad_norm": 0.416015625,
      "learning_rate": 5.926145944483984e-10,
      "logits/chosen": 0.04970569908618927,
      "logits/rejected": 0.41454869508743286,
      "logps/chosen": -197.70941162109375,
      "logps/rejected": -207.9854278564453,
      "loss": 0.6625,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.009294511750340462,
      "rewards/margins": 0.05480729788541794,
      "rewards/margins_max": 0.08153598010540009,
      "rewards/margins_min": 0.02807862125337124,
      "rewards/margins_std": 0.03780006244778633,
      "rewards/rejected": -0.04551279544830322,
      "step": 1690
    },
    {
      "epoch": 0.99,
      "grad_norm": 0.42578125,
      "learning_rate": 2.9534057618091356e-10,
      "logits/chosen": 0.1366875320672989,
      "logits/rejected": 0.4813140034675598,
      "logps/chosen": -195.55368041992188,
      "logps/rejected": -211.63711547851562,
      "loss": 0.6599,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.014302869327366352,
      "rewards/margins": 0.0652112141251564,
      "rewards/margins_max": 0.09685875475406647,
      "rewards/margins_min": 0.03356366977095604,
      "rewards/margins_std": 0.04475637897849083,
      "rewards/rejected": -0.05090833827853203,
      "step": 1700
    },
    {
      "epoch": 0.99,
      "grad_norm": 0.4453125,
      "learning_rate": 1.0051089289686565e-10,
      "logits/chosen": 0.20965194702148438,
      "logits/rejected": 0.5980690121650696,
      "logps/chosen": -218.3548583984375,
      "logps/rejected": -252.60159301757812,
      "loss": 0.6601,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.01929156482219696,
      "rewards/margins": 0.06570716202259064,
      "rewards/margins_max": 0.09711313247680664,
      "rewards/margins_min": 0.03430120274424553,
      "rewards/margins_std": 0.044414736330509186,
      "rewards/rejected": -0.04641559720039368,
      "step": 1710
    },
    {
      "epoch": 1.0,
      "grad_norm": 0.404296875,
      "learning_rate": 8.205475813372054e-12,
      "logits/chosen": 0.07036467641592026,
      "logits/rejected": 0.6885267496109009,
      "logps/chosen": -334.186279296875,
      "logps/rejected": -232.6072998046875,
      "loss": 0.6604,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.015851657837629318,
      "rewards/margins": 0.06690393388271332,
      "rewards/margins_max": 0.0959465354681015,
      "rewards/margins_min": 0.037861332297325134,
      "rewards/margins_std": 0.041072435677051544,
      "rewards/rejected": -0.0510522723197937,
      "step": 1720
    },
    {
      "epoch": 1.0,
      "eval_logits/chosen": 0.7297662496566772,
      "eval_logits/rejected": 0.8997808694839478,
      "eval_logps/chosen": -337.8507080078125,
      "eval_logps/rejected": -318.01556396484375,
      "eval_loss": 0.6928703784942627,
      "eval_rewards/accuracies": 0.5364999771118164,
      "eval_rewards/chosen": 0.002909434260800481,
      "eval_rewards/margins": 0.0005662557086907327,
      "eval_rewards/margins_max": 0.07228709012269974,
      "eval_rewards/margins_min": -0.08225506544113159,
      "eval_rewards/margins_std": 0.050406549125909805,
      "eval_rewards/rejected": 0.002343178726732731,
      "eval_runtime": 864.7602,
      "eval_samples_per_second": 9.251,
      "eval_steps_per_second": 0.289,
      "step": 1724
    },
    {
      "epoch": 1.0,
      "step": 1724,
      "total_flos": 0.0,
      "train_loss": 0.6676546893927447,
      "train_runtime": 9120.8228,
      "train_samples_per_second": 3.024,
      "train_steps_per_second": 0.189
    }
  ],
  "logging_steps": 10,
  "max_steps": 1724,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}