{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.271716015049894,
  "eval_steps": 500,
  "global_step": 2500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 5.000000000000001e-07,
      "logits/chosen": -2.586496591567993,
      "logits/rejected": -2.58866286277771,
      "logps/chosen": -140.71868896484375,
      "logps/rejected": -141.9235382080078,
      "loss": 0.6941,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.005378031637519598,
      "rewards/margins": -0.0017773156287148595,
      "rewards/rejected": -0.0036007165908813477,
      "step": 1
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.0000000000000002e-06,
      "logits/chosen": -2.7250607013702393,
      "logits/rejected": -2.737359046936035,
      "logps/chosen": -239.8214111328125,
      "logps/rejected": -245.04498291015625,
      "loss": 0.69,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0041117193177342415,
      "rewards/margins": 0.006983352825045586,
      "rewards/rejected": -0.002871633041650057,
      "step": 2
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.5e-06,
      "logits/chosen": -2.407137393951416,
      "logits/rejected": -2.3473005294799805,
      "logps/chosen": -184.03599548339844,
      "logps/rejected": -155.21461486816406,
      "loss": 0.6886,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.003687595948576927,
      "rewards/margins": 0.009552741423249245,
      "rewards/rejected": -0.0058651454746723175,
      "step": 3
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.0000000000000003e-06,
      "logits/chosen": -2.7127842903137207,
      "logits/rejected": -2.777282953262329,
      "logps/chosen": -171.6352081298828,
      "logps/rejected": -191.40785217285156,
      "loss": 0.7021,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.004523229319602251,
      "rewards/margins": -0.01729598268866539,
      "rewards/rejected": 0.012772750109434128,
      "step": 4
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.5e-06,
      "logits/chosen": -2.4465034008026123,
      "logits/rejected": -2.466231107711792,
      "logps/chosen": -254.792724609375,
      "logps/rejected": -245.39108276367188,
      "loss": 0.6988,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.004072976298630238,
      "rewards/margins": -0.010413647629320621,
      "rewards/rejected": 0.006340669468045235,
      "step": 5
    },
    {
      "epoch": 0.01,
      "learning_rate": 3e-06,
      "logits/chosen": -2.2196829319000244,
      "logits/rejected": -2.2641539573669434,
      "logps/chosen": -179.39495849609375,
      "logps/rejected": -220.39300537109375,
      "loss": 0.6891,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.01630725897848606,
      "rewards/margins": 0.008536052890121937,
      "rewards/rejected": 0.00777120515704155,
      "step": 6
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.5000000000000004e-06,
      "logits/chosen": -2.3294830322265625,
      "logits/rejected": -2.380467653274536,
      "logps/chosen": -194.7043914794922,
      "logps/rejected": -199.31422424316406,
      "loss": 0.6919,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.005030441097915173,
      "rewards/margins": 0.0030234563164412975,
      "rewards/rejected": 0.0020069843158125877,
      "step": 7
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.000000000000001e-06,
      "logits/chosen": -2.590843677520752,
      "logits/rejected": -2.5107312202453613,
      "logps/chosen": -215.23556518554688,
      "logps/rejected": -189.38424682617188,
      "loss": 0.6995,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0025288108736276627,
      "rewards/margins": -0.012505888007581234,
      "rewards/rejected": 0.009977078065276146,
      "step": 8
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.5e-06,
      "logits/chosen": -2.659663200378418,
      "logits/rejected": -2.6844048500061035,
      "logps/chosen": -229.0231170654297,
      "logps/rejected": -205.54054260253906,
      "loss": 0.7032,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014649391174316406,
      "rewards/margins": -0.019562961533665657,
      "rewards/rejected": 0.0049135684967041016,
      "step": 9
    },
    {
      "epoch": 0.01,
      "learning_rate": 5e-06,
      "logits/chosen": -2.6927154064178467,
      "logits/rejected": -2.6597423553466797,
      "logps/chosen": -207.30467224121094,
      "logps/rejected": -197.54656982421875,
      "loss": 0.692,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0019803522154688835,
      "rewards/margins": 0.002554560313001275,
      "rewards/rejected": -0.004534911829978228,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.500000000000001e-06,
      "logits/chosen": -2.6474952697753906,
      "logits/rejected": -2.6725170612335205,
      "logps/chosen": -231.35108947753906,
      "logps/rejected": -274.9834289550781,
      "loss": 0.6924,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0055945878848433495,
      "rewards/margins": 0.0016048436518758535,
      "rewards/rejected": 0.003989744000136852,
      "step": 11
    },
    {
      "epoch": 0.02,
      "learning_rate": 6e-06,
      "logits/chosen": -2.2198939323425293,
      "logits/rejected": -2.2628731727600098,
      "logps/chosen": -198.29022216796875,
      "logps/rejected": -170.82257080078125,
      "loss": 0.688,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0031775482930243015,
      "rewards/margins": 0.010572671890258789,
      "rewards/rejected": -0.013750219717621803,
      "step": 12
    },
    {
      "epoch": 0.02,
      "learning_rate": 6.5000000000000004e-06,
      "logits/chosen": -2.7613415718078613,
      "logits/rejected": -2.8206562995910645,
      "logps/chosen": -239.75253295898438,
      "logps/rejected": -238.97320556640625,
      "loss": 0.6999,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0029205563478171825,
      "rewards/margins": -0.013042164966464043,
      "rewards/rejected": 0.010121609084308147,
      "step": 13
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.000000000000001e-06,
      "logits/chosen": -2.0942578315734863,
      "logits/rejected": -2.017871856689453,
      "logps/chosen": -184.2721710205078,
      "logps/rejected": -175.73641967773438,
      "loss": 0.6872,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.004038786515593529,
      "rewards/margins": 0.01193075068295002,
      "rewards/rejected": -0.015969539061188698,
      "step": 14
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.5e-06,
      "logits/chosen": -2.526791572570801,
      "logits/rejected": -2.5704824924468994,
      "logps/chosen": -200.25115966796875,
      "logps/rejected": -223.4153289794922,
      "loss": 0.6917,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00700113782659173,
      "rewards/margins": 0.003126000752672553,
      "rewards/rejected": 0.003875136375427246,
      "step": 15
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.000000000000001e-06,
      "logits/chosen": -2.5145041942596436,
      "logits/rejected": -2.478262424468994,
      "logps/chosen": -169.4720001220703,
      "logps/rejected": -179.3622283935547,
      "loss": 0.687,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00018756365170702338,
      "rewards/margins": 0.012972594238817692,
      "rewards/rejected": -0.013160157017409801,
      "step": 16
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.500000000000002e-06,
      "logits/chosen": -2.1660895347595215,
      "logits/rejected": -2.1799678802490234,
      "logps/chosen": -158.1959686279297,
      "logps/rejected": -225.02365112304688,
      "loss": 0.687,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01399233564734459,
      "rewards/margins": 0.01266777515411377,
      "rewards/rejected": 0.001324558281339705,
      "step": 17
    },
    {
      "epoch": 0.02,
      "learning_rate": 9e-06,
      "logits/chosen": -2.417539596557617,
      "logits/rejected": -2.4626646041870117,
      "logps/chosen": -170.4958038330078,
      "logps/rejected": -178.34300231933594,
      "loss": 0.6877,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0049651628360152245,
      "rewards/margins": 0.011217641644179821,
      "rewards/rejected": -0.006252479739487171,
      "step": 18
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.5e-06,
      "logits/chosen": -2.6470260620117188,
      "logits/rejected": -2.7213802337646484,
      "logps/chosen": -179.53497314453125,
      "logps/rejected": -200.94956970214844,
      "loss": 0.69,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0027004480361938477,
      "rewards/margins": 0.006518864538520575,
      "rewards/rejected": -0.00921931304037571,
      "step": 19
    },
    {
      "epoch": 0.03,
      "learning_rate": 1e-05,
      "logits/chosen": -2.4962029457092285,
      "logits/rejected": -2.5313973426818848,
      "logps/chosen": -208.752685546875,
      "logps/rejected": -244.1136474609375,
      "loss": 0.683,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.009996438398957253,
      "rewards/margins": 0.020868420600891113,
      "rewards/rejected": -0.01087198406457901,
      "step": 20
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.05e-05,
      "logits/chosen": -2.531207799911499,
      "logits/rejected": -2.5359435081481934,
      "logps/chosen": -194.47694396972656,
      "logps/rejected": -185.80142211914062,
      "loss": 0.6895,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.002445125486701727,
      "rewards/margins": 0.0077151767909526825,
      "rewards/rejected": -0.005270051304250956,
      "step": 21
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.1000000000000001e-05,
      "logits/chosen": -2.430760622024536,
      "logits/rejected": -2.395613431930542,
      "logps/chosen": -144.2230987548828,
      "logps/rejected": -148.8902587890625,
      "loss": 0.6942,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00968785211443901,
      "rewards/margins": -0.001990032149478793,
      "rewards/rejected": -0.007697821129113436,
      "step": 22
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.1500000000000002e-05,
      "logits/chosen": -2.4417223930358887,
      "logits/rejected": -2.441425323486328,
      "logps/chosen": -165.82875061035156,
      "logps/rejected": -178.82815551757812,
      "loss": 0.6886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0060547590255737305,
      "rewards/margins": 0.009464193135499954,
      "rewards/rejected": -0.0034094336442649364,
      "step": 23
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2e-05,
      "logits/chosen": -2.3494763374328613,
      "logits/rejected": -2.40555477142334,
      "logps/chosen": -133.24130249023438,
      "logps/rejected": -151.85446166992188,
      "loss": 0.6986,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.011995697394013405,
      "rewards/margins": -0.010638857260346413,
      "rewards/rejected": -0.0013568403664976358,
      "step": 24
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.25e-05,
      "logits/chosen": -2.3450560569763184,
      "logits/rejected": -2.5124807357788086,
      "logps/chosen": -242.19821166992188,
      "logps/rejected": -259.1798095703125,
      "loss": 0.6912,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.013580609112977982,
      "rewards/margins": 0.004313135519623756,
      "rewards/rejected": -0.017893744632601738,
      "step": 25
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3000000000000001e-05,
      "logits/chosen": -2.5465288162231445,
      "logits/rejected": -2.5980730056762695,
      "logps/chosen": -202.898193359375,
      "logps/rejected": -248.03968811035156,
      "loss": 0.6854,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0018544672057032585,
      "rewards/margins": 0.016238166019320488,
      "rewards/rejected": -0.01438369695097208,
      "step": 26
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.3500000000000001e-05,
      "logits/chosen": -2.733663320541382,
      "logits/rejected": -2.712355375289917,
      "logps/chosen": -205.80429077148438,
      "logps/rejected": -174.95797729492188,
      "loss": 0.6933,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -8.790497668087482e-05,
      "rewards/margins": 2.1266518160700798e-05,
      "rewards/rejected": -0.0001091718440875411,
      "step": 27
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.4000000000000001e-05,
      "logits/chosen": -2.550997257232666,
      "logits/rejected": -2.4390013217926025,
      "logps/chosen": -171.59786987304688,
      "logps/rejected": -176.4530029296875,
      "loss": 0.6931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02987699769437313,
      "rewards/margins": 0.0004779808223247528,
      "rewards/rejected": -0.030354974791407585,
      "step": 28
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.45e-05,
      "logits/chosen": -2.4879837036132812,
      "logits/rejected": -2.480923652648926,
      "logps/chosen": -149.10079956054688,
      "logps/rejected": -162.999267578125,
      "loss": 0.6834,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.005112767685204744,
      "rewards/margins": 0.01993861421942711,
      "rewards/rejected": -0.025051379576325417,
      "step": 29
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.5e-05,
      "logits/chosen": -2.3427562713623047,
      "logits/rejected": -2.5123889446258545,
      "logps/chosen": -160.62950134277344,
      "logps/rejected": -201.14747619628906,
      "loss": 0.6952,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.015586448833346367,
      "rewards/margins": -0.003298282390460372,
      "rewards/rejected": -0.012288165278732777,
      "step": 30
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.55e-05,
      "logits/chosen": -2.2548415660858154,
      "logits/rejected": -2.3827996253967285,
      "logps/chosen": -126.6205825805664,
      "logps/rejected": -145.7274627685547,
      "loss": 0.6883,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02384941652417183,
      "rewards/margins": 0.010512137785553932,
      "rewards/rejected": -0.03436155617237091,
      "step": 31
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.6000000000000003e-05,
      "logits/chosen": -2.5090818405151367,
      "logits/rejected": -2.560624122619629,
      "logps/chosen": -182.53944396972656,
      "logps/rejected": -208.08763122558594,
      "loss": 0.6877,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0014843230601400137,
      "rewards/margins": 0.011496281251311302,
      "rewards/rejected": -0.01298060454428196,
      "step": 32
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.65e-05,
      "logits/chosen": -2.800107479095459,
      "logits/rejected": -2.7355594635009766,
      "logps/chosen": -207.99392700195312,
      "logps/rejected": -208.31610107421875,
      "loss": 0.6927,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.014949416741728783,
      "rewards/margins": 0.0012244456447660923,
      "rewards/rejected": -0.016173863783478737,
      "step": 33
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.7000000000000003e-05,
      "logits/chosen": -2.6325690746307373,
      "logits/rejected": -2.630256175994873,
      "logps/chosen": -196.8790283203125,
      "logps/rejected": -202.53794860839844,
      "loss": 0.7116,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.041500113904476166,
      "rewards/margins": -0.03608906269073486,
      "rewards/rejected": -0.005411052145063877,
      "step": 34
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.75e-05,
      "logits/chosen": -2.476334571838379,
      "logits/rejected": -2.473987579345703,
      "logps/chosen": -256.1812438964844,
      "logps/rejected": -250.76165771484375,
      "loss": 0.6882,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.01104288175702095,
      "rewards/margins": 0.010350894182920456,
      "rewards/rejected": -0.021393775939941406,
      "step": 35
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.8e-05,
      "logits/chosen": -2.342329502105713,
      "logits/rejected": -2.3805429935455322,
      "logps/chosen": -164.03692626953125,
      "logps/rejected": -217.39849853515625,
      "loss": 0.688,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.009070659056305885,
      "rewards/margins": 0.010929775424301624,
      "rewards/rejected": -0.020000435411930084,
      "step": 36
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.85e-05,
      "logits/chosen": -2.4253318309783936,
      "logits/rejected": -2.4550042152404785,
      "logps/chosen": -146.70420837402344,
      "logps/rejected": -129.80674743652344,
      "loss": 0.6845,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.031095195561647415,
      "rewards/margins": 0.017679547891020775,
      "rewards/rejected": -0.04877474159002304,
      "step": 37
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.9e-05,
      "logits/chosen": -2.4936635494232178,
      "logits/rejected": -2.5476386547088623,
      "logps/chosen": -164.6866455078125,
      "logps/rejected": -198.52139282226562,
      "loss": 0.6895,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.025130080059170723,
      "rewards/margins": 0.007643342949450016,
      "rewards/rejected": -0.03277342766523361,
      "step": 38
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.9500000000000003e-05,
      "logits/chosen": -2.4614553451538086,
      "logits/rejected": -2.4693663120269775,
      "logps/chosen": -172.24966430664062,
      "logps/rejected": -166.52056884765625,
      "loss": 0.6975,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.027715325355529785,
      "rewards/margins": -0.00841212272644043,
      "rewards/rejected": -0.019303202629089355,
      "step": 39
    },
    {
      "epoch": 0.05,
      "learning_rate": 2e-05,
      "logits/chosen": -2.3024024963378906,
      "logits/rejected": -2.2906301021575928,
      "logps/chosen": -243.98683166503906,
      "logps/rejected": -199.12408447265625,
      "loss": 0.6979,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.028235863894224167,
      "rewards/margins": -0.00847182422876358,
      "rewards/rejected": -0.019764041528105736,
      "step": 40
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.05e-05,
      "logits/chosen": -2.660562753677368,
      "logits/rejected": -2.6754074096679688,
      "logps/chosen": -156.67514038085938,
      "logps/rejected": -147.5068359375,
      "loss": 0.6887,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03220677375793457,
      "rewards/margins": 0.009226083755493164,
      "rewards/rejected": -0.041432857513427734,
      "step": 41
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.1e-05,
      "logits/chosen": -2.482787847518921,
      "logits/rejected": -2.4663233757019043,
      "logps/chosen": -207.6859588623047,
      "logps/rejected": -182.35931396484375,
      "loss": 0.6875,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.014715791679918766,
      "rewards/margins": 0.012153576128184795,
      "rewards/rejected": -0.02686937153339386,
      "step": 42
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.15e-05,
      "logits/chosen": -2.6544148921966553,
      "logits/rejected": -2.717411756515503,
      "logps/chosen": -197.96400451660156,
      "logps/rejected": -223.0219268798828,
      "loss": 0.691,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.024796580895781517,
      "rewards/margins": 0.005184031091630459,
      "rewards/rejected": -0.0299806110560894,
      "step": 43
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.2000000000000003e-05,
      "logits/chosen": -2.561187744140625,
      "logits/rejected": -2.6224913597106934,
      "logps/chosen": -194.60935974121094,
      "logps/rejected": -194.01577758789062,
      "loss": 0.6971,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.020246006548404694,
      "rewards/margins": -0.0076716188341379166,
      "rewards/rejected": -0.012574386782944202,
      "step": 44
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.25e-05,
      "logits/chosen": -2.5130224227905273,
      "logits/rejected": -2.672990083694458,
      "logps/chosen": -154.0092010498047,
      "logps/rejected": -190.70748901367188,
      "loss": 0.6818,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.018519926816225052,
      "rewards/margins": 0.023535681888461113,
      "rewards/rejected": -0.042055610567331314,
      "step": 45
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.3000000000000003e-05,
      "logits/chosen": -2.738673686981201,
      "logits/rejected": -2.82279896736145,
      "logps/chosen": -174.6641387939453,
      "logps/rejected": -163.0157928466797,
      "loss": 0.6967,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014652942307293415,
      "rewards/margins": -0.006221937946975231,
      "rewards/rejected": -0.008431006222963333,
      "step": 46
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.35e-05,
      "logits/chosen": -2.581393241882324,
      "logits/rejected": -2.5734691619873047,
      "logps/chosen": -137.8702392578125,
      "logps/rejected": -154.74896240234375,
      "loss": 0.6887,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.007426738273352385,
      "rewards/margins": 0.009324884042143822,
      "rewards/rejected": -0.016751624643802643,
      "step": 47
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.4e-05,
      "logits/chosen": -2.544344425201416,
      "logits/rejected": -2.690079689025879,
      "logps/chosen": -185.8907012939453,
      "logps/rejected": -195.9602508544922,
      "loss": 0.6898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02690129354596138,
      "rewards/margins": 0.007246016524732113,
      "rewards/rejected": -0.03414731100201607,
      "step": 48
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.45e-05,
      "logits/chosen": -2.583885669708252,
      "logits/rejected": -2.609614849090576,
      "logps/chosen": -215.93264770507812,
      "logps/rejected": -234.49636840820312,
      "loss": 0.6858,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.022774625569581985,
      "rewards/margins": 0.015723586082458496,
      "rewards/rejected": -0.03849821165204048,
      "step": 49
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.5e-05,
      "logits/chosen": -2.6080195903778076,
      "logits/rejected": -2.655003070831299,
      "logps/chosen": -229.5977020263672,
      "logps/rejected": -211.47779846191406,
      "loss": 0.6925,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.026349734514951706,
      "rewards/margins": 0.0026414887979626656,
      "rewards/rejected": -0.028991222381591797,
      "step": 50
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.5500000000000003e-05,
      "logits/chosen": -2.180418014526367,
      "logits/rejected": -2.210296869277954,
      "logps/chosen": -221.1846466064453,
      "logps/rejected": -185.85655212402344,
      "loss": 0.6891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013596129603683949,
      "rewards/margins": 0.009130668826401234,
      "rewards/rejected": -0.022726796567440033,
      "step": 51
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.6000000000000002e-05,
      "logits/chosen": -2.647798538208008,
      "logits/rejected": -2.715623617172241,
      "logps/chosen": -186.01138305664062,
      "logps/rejected": -180.89259338378906,
      "loss": 0.6785,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.019299650564789772,
      "rewards/margins": 0.030588869005441666,
      "rewards/rejected": -0.011289214715361595,
      "step": 52
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.6500000000000004e-05,
      "logits/chosen": -2.570209503173828,
      "logits/rejected": -2.5728917121887207,
      "logps/chosen": -209.2962188720703,
      "logps/rejected": -216.60145568847656,
      "loss": 0.6824,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.005204535089433193,
      "rewards/margins": 0.021862652152776718,
      "rewards/rejected": -0.027067184448242188,
      "step": 53
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.7000000000000002e-05,
      "logits/chosen": -2.672147512435913,
      "logits/rejected": -2.678018808364868,
      "logps/chosen": -167.4713134765625,
      "logps/rejected": -170.08120727539062,
      "loss": 0.691,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013661455363035202,
      "rewards/margins": 0.004836535546928644,
      "rewards/rejected": -0.018497992306947708,
      "step": 54
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.7500000000000004e-05,
      "logits/chosen": -2.4589505195617676,
      "logits/rejected": -2.462191343307495,
      "logps/chosen": -200.73976135253906,
      "logps/rejected": -195.30276489257812,
      "loss": 0.704,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03748317062854767,
      "rewards/margins": -0.020142268389463425,
      "rewards/rejected": -0.017340898513793945,
      "step": 55
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.8000000000000003e-05,
      "logits/chosen": -2.230989933013916,
      "logits/rejected": -2.2140636444091797,
      "logps/chosen": -143.33631896972656,
      "logps/rejected": -136.0796661376953,
      "loss": 0.6989,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.020144915208220482,
      "rewards/margins": -0.01022801361978054,
      "rewards/rejected": -0.009916901588439941,
      "step": 56
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.8499999999999998e-05,
      "logits/chosen": -2.5017194747924805,
      "logits/rejected": -2.58662486076355,
      "logps/chosen": -167.50990295410156,
      "logps/rejected": -208.83607482910156,
      "loss": 0.6808,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0027861359994858503,
      "rewards/margins": 0.02604994922876358,
      "rewards/rejected": -0.02883608266711235,
      "step": 57
    },
    {
      "epoch": 0.08,
      "learning_rate": 2.9e-05,
      "logits/chosen": -2.53066349029541,
      "logits/rejected": -2.608471393585205,
      "logps/chosen": -241.85113525390625,
      "logps/rejected": -260.0408630371094,
      "loss": 0.6939,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.024016117677092552,
      "rewards/margins": -0.0005100721027702093,
      "rewards/rejected": -0.023506049066781998,
      "step": 58
    },
    {
      "epoch": 0.08,
      "learning_rate": 2.95e-05,
      "logits/chosen": -2.2705140113830566,
      "logits/rejected": -2.4387779235839844,
      "logps/chosen": -179.22540283203125,
      "logps/rejected": -182.5544891357422,
      "loss": 0.6775,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.00418963422998786,
      "rewards/margins": 0.033549048006534576,
      "rewards/rejected": -0.02935941144824028,
      "step": 59
    },
    {
      "epoch": 0.08,
      "learning_rate": 3e-05,
      "logits/chosen": -2.46372652053833,
      "logits/rejected": -2.513279438018799,
      "logps/chosen": -230.43133544921875,
      "logps/rejected": -227.0118865966797,
      "loss": 0.6761,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.010919665917754173,
      "rewards/margins": 0.03570995107293129,
      "rewards/rejected": -0.04662961885333061,
      "step": 60
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.05e-05,
      "logits/chosen": -2.530576467514038,
      "logits/rejected": -2.5415897369384766,
      "logps/chosen": -200.46568298339844,
      "logps/rejected": -220.82989501953125,
      "loss": 0.6794,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0384901762008667,
      "rewards/margins": 0.03156058490276337,
      "rewards/rejected": -0.07005076855421066,
      "step": 61
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.1e-05,
      "logits/chosen": -2.5888161659240723,
      "logits/rejected": -2.639291286468506,
      "logps/chosen": -132.55116271972656,
      "logps/rejected": -167.082275390625,
      "loss": 0.7042,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.03156990930438042,
      "rewards/margins": -0.018804360181093216,
      "rewards/rejected": -0.0127655528485775,
      "step": 62
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.15e-05,
      "logits/chosen": -2.46744704246521,
      "logits/rejected": -2.494837522506714,
      "logps/chosen": -210.9451141357422,
      "logps/rejected": -211.72433471679688,
      "loss": 0.6799,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0017502065747976303,
      "rewards/margins": 0.027423406019806862,
      "rewards/rejected": -0.02567320130765438,
      "step": 63
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.2000000000000005e-05,
      "logits/chosen": -2.4783239364624023,
      "logits/rejected": -2.490576982498169,
      "logps/chosen": -197.71629333496094,
      "logps/rejected": -207.23048400878906,
      "loss": 0.6864,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.01412363164126873,
      "rewards/margins": 0.01561451144516468,
      "rewards/rejected": -0.029738139361143112,
      "step": 64
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.2500000000000004e-05,
      "logits/chosen": -2.3918569087982178,
      "logits/rejected": -2.4426636695861816,
      "logps/chosen": -164.30029296875,
      "logps/rejected": -201.85992431640625,
      "loss": 0.6908,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.025302361696958542,
      "rewards/margins": 0.009253643453121185,
      "rewards/rejected": -0.034556008875370026,
      "step": 65
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.3e-05,
      "logits/chosen": -2.4934239387512207,
      "logits/rejected": -2.5188210010528564,
      "logps/chosen": -178.12030029296875,
      "logps/rejected": -211.23947143554688,
      "loss": 0.6908,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.05651288107037544,
      "rewards/margins": 0.0065690516494214535,
      "rewards/rejected": -0.06308193504810333,
      "step": 66
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.35e-05,
      "logits/chosen": -2.415985345840454,
      "logits/rejected": -2.3687984943389893,
      "logps/chosen": -211.91453552246094,
      "logps/rejected": -216.65591430664062,
      "loss": 0.6808,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.045519113540649414,
      "rewards/margins": 0.028419354930520058,
      "rewards/rejected": -0.07393846660852432,
      "step": 67
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.4000000000000007e-05,
      "logits/chosen": -2.37481427192688,
      "logits/rejected": -2.4693892002105713,
      "logps/chosen": -172.92459106445312,
      "logps/rejected": -209.3857421875,
      "loss": 0.6801,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03707296401262283,
      "rewards/margins": 0.027949143201112747,
      "rewards/rejected": -0.06502211093902588,
      "step": 68
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.45e-05,
      "logits/chosen": -2.3615283966064453,
      "logits/rejected": -2.3259239196777344,
      "logps/chosen": -154.25360107421875,
      "logps/rejected": -161.130126953125,
      "loss": 0.7022,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.04533124342560768,
      "rewards/margins": -0.016323519870638847,
      "rewards/rejected": -0.029007721692323685,
      "step": 69
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.5e-05,
      "logits/chosen": -2.64551043510437,
      "logits/rejected": -2.7289645671844482,
      "logps/chosen": -200.1268768310547,
      "logps/rejected": -225.6541748046875,
      "loss": 0.6993,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0578455924987793,
      "rewards/margins": -0.008899472653865814,
      "rewards/rejected": -0.04894612357020378,
      "step": 70
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.55e-05,
      "logits/chosen": -2.703721046447754,
      "logits/rejected": -2.669755220413208,
      "logps/chosen": -208.99668884277344,
      "logps/rejected": -164.8292694091797,
      "loss": 0.6756,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.022734597325325012,
      "rewards/margins": 0.03882308304309845,
      "rewards/rejected": -0.06155767664313316,
      "step": 71
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.6e-05,
      "logits/chosen": -2.6480765342712402,
      "logits/rejected": -2.696770191192627,
      "logps/chosen": -172.99888610839844,
      "logps/rejected": -201.09979248046875,
      "loss": 0.7063,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.053855180740356445,
      "rewards/margins": -0.024987507611513138,
      "rewards/rejected": -0.028867674991488457,
      "step": 72
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.65e-05,
      "logits/chosen": -2.6311445236206055,
      "logits/rejected": -2.6575632095336914,
      "logps/chosen": -251.72789001464844,
      "logps/rejected": -271.0470886230469,
      "loss": 0.6867,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03289387375116348,
      "rewards/margins": 0.014702942222356796,
      "rewards/rejected": -0.04759681224822998,
      "step": 73
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.7e-05,
      "logits/chosen": -2.3990683555603027,
      "logits/rejected": -2.502671480178833,
      "logps/chosen": -197.92665100097656,
      "logps/rejected": -211.00967407226562,
      "loss": 0.6841,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.014272330328822136,
      "rewards/margins": 0.020740672945976257,
      "rewards/rejected": -0.03501300886273384,
      "step": 74
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.7500000000000003e-05,
      "logits/chosen": -2.575502872467041,
      "logits/rejected": -2.527902364730835,
      "logps/chosen": -214.5215301513672,
      "logps/rejected": -201.05194091796875,
      "loss": 0.6651,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.006246686447411776,
      "rewards/margins": 0.059338975697755814,
      "rewards/rejected": -0.065585657954216,
      "step": 75
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.8e-05,
      "logits/chosen": -2.217294692993164,
      "logits/rejected": -2.223574638366699,
      "logps/chosen": -174.7819366455078,
      "logps/rejected": -178.09852600097656,
      "loss": 0.7013,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09115451574325562,
      "rewards/margins": -0.012327454052865505,
      "rewards/rejected": -0.07882705330848694,
      "step": 76
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.85e-05,
      "logits/chosen": -2.518965721130371,
      "logits/rejected": -2.443446159362793,
      "logps/chosen": -201.1251220703125,
      "logps/rejected": -170.15533447265625,
      "loss": 0.7029,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04162323474884033,
      "rewards/margins": -0.015459035523235798,
      "rewards/rejected": -0.02616419643163681,
      "step": 77
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.9000000000000006e-05,
      "logits/chosen": -2.521008253097534,
      "logits/rejected": -2.6866934299468994,
      "logps/chosen": -269.85284423828125,
      "logps/rejected": -240.82626342773438,
      "loss": 0.6928,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.05473289266228676,
      "rewards/margins": 0.00423135980963707,
      "rewards/rejected": -0.05896425247192383,
      "step": 78
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.9500000000000005e-05,
      "logits/chosen": -2.3015501499176025,
      "logits/rejected": -2.351773500442505,
      "logps/chosen": -225.17691040039062,
      "logps/rejected": -249.8846435546875,
      "loss": 0.6873,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.06839534640312195,
      "rewards/margins": 0.014748764224350452,
      "rewards/rejected": -0.08314411342144012,
      "step": 79
    },
    {
      "epoch": 0.1,
      "learning_rate": 4e-05,
      "logits/chosen": -2.6201720237731934,
      "logits/rejected": -2.680964708328247,
      "logps/chosen": -225.08645629882812,
      "logps/rejected": -232.00096130371094,
      "loss": 0.6739,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06486918777227402,
      "rewards/margins": 0.04111311584711075,
      "rewards/rejected": -0.10598230361938477,
      "step": 80
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.05e-05,
      "logits/chosen": -2.5782110691070557,
      "logits/rejected": -2.5646634101867676,
      "logps/chosen": -249.90745544433594,
      "logps/rejected": -257.51275634765625,
      "loss": 0.6976,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08720846474170685,
      "rewards/margins": -0.0017095585353672504,
      "rewards/rejected": -0.08549890667200089,
      "step": 81
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.1e-05,
      "logits/chosen": -2.5777688026428223,
      "logits/rejected": -2.665797472000122,
      "logps/chosen": -171.82347106933594,
      "logps/rejected": -211.63311767578125,
      "loss": 0.6883,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08461908996105194,
      "rewards/margins": 0.013853237964212894,
      "rewards/rejected": -0.09847234189510345,
      "step": 82
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.15e-05,
      "logits/chosen": -2.3364946842193604,
      "logits/rejected": -2.3084681034088135,
      "logps/chosen": -193.4939727783203,
      "logps/rejected": -184.15200805664062,
      "loss": 0.717,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0981694683432579,
      "rewards/margins": -0.04480106756091118,
      "rewards/rejected": -0.053368404507637024,
      "step": 83
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.2e-05,
      "logits/chosen": -2.445204257965088,
      "logits/rejected": -2.5341625213623047,
      "logps/chosen": -174.59225463867188,
      "logps/rejected": -210.94137573242188,
      "loss": 0.699,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.09675168246030807,
      "rewards/margins": -0.00985276885330677,
      "rewards/rejected": -0.08689891546964645,
      "step": 84
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.25e-05,
      "logits/chosen": -2.4798362255096436,
      "logits/rejected": -2.4760632514953613,
      "logps/chosen": -148.0768585205078,
      "logps/rejected": -176.31700134277344,
      "loss": 0.6945,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.07728591561317444,
      "rewards/margins": 0.0020556673407554626,
      "rewards/rejected": -0.0793415829539299,
      "step": 85
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.3e-05,
      "logits/chosen": -2.5911381244659424,
      "logits/rejected": -2.630659341812134,
      "logps/chosen": -203.1630859375,
      "logps/rejected": -183.4739227294922,
      "loss": 0.653,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.04599933326244354,
      "rewards/margins": 0.08558819442987442,
      "rewards/rejected": -0.13158753514289856,
      "step": 86
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.35e-05,
      "logits/chosen": -2.600797414779663,
      "logits/rejected": -2.6318492889404297,
      "logps/chosen": -156.8225860595703,
      "logps/rejected": -161.6145477294922,
      "loss": 0.6705,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07066044956445694,
      "rewards/margins": 0.05500438064336777,
      "rewards/rejected": -0.1256648302078247,
      "step": 87
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.4000000000000006e-05,
      "logits/chosen": -2.2124781608581543,
      "logits/rejected": -2.259019374847412,
      "logps/chosen": -142.0679473876953,
      "logps/rejected": -177.9369659423828,
      "loss": 0.6949,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13633377850055695,
      "rewards/margins": 0.000554969534277916,
      "rewards/rejected": -0.1368887573480606,
      "step": 88
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.4500000000000004e-05,
      "logits/chosen": -2.396183729171753,
      "logits/rejected": -2.533904552459717,
      "logps/chosen": -171.38790893554688,
      "logps/rejected": -202.30979919433594,
      "loss": 0.6554,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09015192836523056,
      "rewards/margins": 0.08280421048402786,
      "rewards/rejected": -0.17295613884925842,
      "step": 89
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.5e-05,
      "logits/chosen": -2.640425443649292,
      "logits/rejected": -2.674539804458618,
      "logps/chosen": -175.3508758544922,
      "logps/rejected": -180.97915649414062,
      "loss": 0.6768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1077599823474884,
      "rewards/margins": 0.03759467601776123,
      "rewards/rejected": -0.14535464346408844,
      "step": 90
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.55e-05,
      "logits/chosen": -2.5857696533203125,
      "logits/rejected": -2.5822975635528564,
      "logps/chosen": -176.55982971191406,
      "logps/rejected": -196.5354461669922,
      "loss": 0.7,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.09103889763355255,
      "rewards/margins": -0.009319041855633259,
      "rewards/rejected": -0.08171986043453217,
      "step": 91
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.600000000000001e-05,
      "logits/chosen": -2.660163402557373,
      "logits/rejected": -2.5935940742492676,
      "logps/chosen": -233.04049682617188,
      "logps/rejected": -212.1655731201172,
      "loss": 0.6863,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.118477463722229,
      "rewards/margins": 0.017601586878299713,
      "rewards/rejected": -0.13607905805110931,
      "step": 92
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.6500000000000005e-05,
      "logits/chosen": -2.5211241245269775,
      "logits/rejected": -2.6064364910125732,
      "logps/chosen": -176.89476013183594,
      "logps/rejected": -187.39271545410156,
      "loss": 0.6736,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07700353115797043,
      "rewards/margins": 0.04530329257249832,
      "rewards/rejected": -0.12230683863162994,
      "step": 93
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.7e-05,
      "logits/chosen": -2.6894078254699707,
      "logits/rejected": -2.6806745529174805,
      "logps/chosen": -244.7545623779297,
      "logps/rejected": -212.83453369140625,
      "loss": 0.6446,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10891007632017136,
      "rewards/margins": 0.10942523181438446,
      "rewards/rejected": -0.2183353304862976,
      "step": 94
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.75e-05,
      "logits/chosen": -2.4758429527282715,
      "logits/rejected": -2.5564448833465576,
      "logps/chosen": -161.29554748535156,
      "logps/rejected": -185.48199462890625,
      "loss": 0.6991,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.152861088514328,
      "rewards/margins": -0.0035788798704743385,
      "rewards/rejected": -0.14928221702575684,
      "step": 95
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.8e-05,
      "logits/chosen": -2.47318959236145,
      "logits/rejected": -2.439746856689453,
      "logps/chosen": -241.09283447265625,
      "logps/rejected": -218.68580627441406,
      "loss": 0.7017,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1952984780073166,
      "rewards/margins": -0.01051153801381588,
      "rewards/rejected": -0.18478693068027496,
      "step": 96
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.85e-05,
      "logits/chosen": -2.541466474533081,
      "logits/rejected": -2.56956148147583,
      "logps/chosen": -180.84725952148438,
      "logps/rejected": -211.02352905273438,
      "loss": 0.6627,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15894030034542084,
      "rewards/margins": 0.06507530808448792,
      "rewards/rejected": -0.22401559352874756,
      "step": 97
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9e-05,
      "logits/chosen": -2.226611852645874,
      "logits/rejected": -2.359558343887329,
      "logps/chosen": -145.781494140625,
      "logps/rejected": -181.21531677246094,
      "loss": 0.6689,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.177232027053833,
      "rewards/margins": 0.058251187205314636,
      "rewards/rejected": -0.23548321425914764,
      "step": 98
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9500000000000004e-05,
      "logits/chosen": -2.5605039596557617,
      "logits/rejected": -2.641690492630005,
      "logps/chosen": -156.43446350097656,
      "logps/rejected": -171.6468505859375,
      "loss": 0.6801,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.23164159059524536,
      "rewards/margins": 0.03432049974799156,
      "rewards/rejected": -0.2659620940685272,
      "step": 99
    },
    {
      "epoch": 0.13,
      "learning_rate": 5e-05,
      "logits/chosen": -2.7840309143066406,
      "logits/rejected": -2.8028616905212402,
      "logps/chosen": -221.6836395263672,
      "logps/rejected": -247.433349609375,
      "loss": 0.647,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.14767608046531677,
      "rewards/margins": 0.1068597286939621,
      "rewards/rejected": -0.25453582406044006,
      "step": 100
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.99999978299634e-05,
      "logits/chosen": -2.1325531005859375,
      "logits/rejected": -2.1438934803009033,
      "logps/chosen": -171.369140625,
      "logps/rejected": -184.96827697753906,
      "loss": 0.6417,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.18850459158420563,
      "rewards/margins": 0.12018898129463196,
      "rewards/rejected": -0.3086935877799988,
      "step": 101
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.999999131985394e-05,
      "logits/chosen": -2.6781508922576904,
      "logits/rejected": -2.5359549522399902,
      "logps/chosen": -232.26742553710938,
      "logps/rejected": -214.5188446044922,
      "loss": 0.7372,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.32363808155059814,
      "rewards/margins": -0.07766950130462646,
      "rewards/rejected": -0.2459685504436493,
      "step": 102
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.999998046967279e-05,
      "logits/chosen": -2.54486346244812,
      "logits/rejected": -2.6447086334228516,
      "logps/chosen": -179.6219482421875,
      "logps/rejected": -222.37010192871094,
      "loss": 0.6585,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23128211498260498,
      "rewards/margins": 0.08162947744131088,
      "rewards/rejected": -0.31291159987449646,
      "step": 103
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9999965279421804e-05,
      "logits/chosen": -2.4281625747680664,
      "logits/rejected": -2.5068893432617188,
      "logps/chosen": -163.3339385986328,
      "logps/rejected": -180.06222534179688,
      "loss": 0.6739,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22076301276683807,
      "rewards/margins": 0.05238480493426323,
      "rewards/rejected": -0.2731478214263916,
      "step": 104
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.999994574910364e-05,
      "logits/chosen": -2.5497522354125977,
      "logits/rejected": -2.558523178100586,
      "logps/chosen": -214.64537048339844,
      "logps/rejected": -210.29678344726562,
      "loss": 0.6691,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19515390694141388,
      "rewards/margins": 0.06168302148580551,
      "rewards/rejected": -0.2568369209766388,
      "step": 105
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.999992187872167e-05,
      "logits/chosen": -2.6358773708343506,
      "logits/rejected": -2.672025680541992,
      "logps/chosen": -235.18109130859375,
      "logps/rejected": -233.78045654296875,
      "loss": 0.7203,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.25453630089759827,
      "rewards/margins": -0.03941688686609268,
      "rewards/rejected": -0.215119406580925,
      "step": 106
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9999893668280043e-05,
      "logits/chosen": -2.305128812789917,
      "logits/rejected": -2.384552478790283,
      "logps/chosen": -168.1253662109375,
      "logps/rejected": -210.5357666015625,
      "loss": 0.6558,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.30615508556365967,
      "rewards/margins": 0.10809167474508286,
      "rewards/rejected": -0.4142467677593231,
      "step": 107
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.999986111778367e-05,
      "logits/chosen": -2.4207632541656494,
      "logits/rejected": -2.4426565170288086,
      "logps/chosen": -187.8740234375,
      "logps/rejected": -188.6339569091797,
      "loss": 0.6383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2796909809112549,
      "rewards/margins": 0.13547499477863312,
      "rewards/rejected": -0.4151660203933716,
      "step": 108
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.999982422723818e-05,
      "logits/chosen": -2.505187749862671,
      "logits/rejected": -2.4738471508026123,
      "logps/chosen": -187.51710510253906,
      "logps/rejected": -227.13916015625,
      "loss": 0.7216,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5267550945281982,
      "rewards/margins": -0.03421187028288841,
      "rewards/rejected": -0.4925432801246643,
      "step": 109
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9999782996649994e-05,
      "logits/chosen": -2.4367451667785645,
      "logits/rejected": -2.4629135131835938,
      "logps/chosen": -221.5149688720703,
      "logps/rejected": -255.66635131835938,
      "loss": 0.6923,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4186938405036926,
      "rewards/margins": 0.03650692105293274,
      "rewards/rejected": -0.45520079135894775,
      "step": 110
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.999973742602626e-05,
      "logits/chosen": -2.6166908740997314,
      "logits/rejected": -2.638493061065674,
      "logps/chosen": -212.32894897460938,
      "logps/rejected": -216.49720764160156,
      "loss": 0.6789,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.47340863943099976,
      "rewards/margins": 0.059217117726802826,
      "rewards/rejected": -0.5326257944107056,
      "step": 111
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.999968751537489e-05,
      "logits/chosen": -2.549008846282959,
      "logits/rejected": -2.6095943450927734,
      "logps/chosen": -205.5695343017578,
      "logps/rejected": -196.99383544921875,
      "loss": 0.7338,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.47608208656311035,
      "rewards/margins": -0.05748309940099716,
      "rewards/rejected": -0.4185989201068878,
      "step": 112
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9999633264704564e-05,
      "logits/chosen": -2.2947440147399902,
      "logits/rejected": -2.240459680557251,
      "logps/chosen": -203.31182861328125,
      "logps/rejected": -175.49354553222656,
      "loss": 0.7139,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.39874717593193054,
      "rewards/margins": -0.014322709292173386,
      "rewards/rejected": -0.38442447781562805,
      "step": 113
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.999957467402468e-05,
      "logits/chosen": -2.118520736694336,
      "logits/rejected": -2.0125582218170166,
      "logps/chosen": -159.09979248046875,
      "logps/rejected": -151.80679321289062,
      "loss": 0.6981,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.43373599648475647,
      "rewards/margins": 0.0064653148874640465,
      "rewards/rejected": -0.4402012825012207,
      "step": 114
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9999511743345426e-05,
      "logits/chosen": -2.251662254333496,
      "logits/rejected": -2.343283176422119,
      "logps/chosen": -166.46377563476562,
      "logps/rejected": -213.02813720703125,
      "loss": 0.6281,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.44466906785964966,
      "rewards/margins": 0.18120086193084717,
      "rewards/rejected": -0.625869870185852,
      "step": 115
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.999944447267771e-05,
      "logits/chosen": -2.347130298614502,
      "logits/rejected": -2.342409610748291,
      "logps/chosen": -216.30650329589844,
      "logps/rejected": -210.64273071289062,
      "loss": 0.7966,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6098219752311707,
      "rewards/margins": -0.12615808844566345,
      "rewards/rejected": -0.4836638867855072,
      "step": 116
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.999937286203322e-05,
      "logits/chosen": -2.3265388011932373,
      "logits/rejected": -2.3534939289093018,
      "logps/chosen": -204.47706604003906,
      "logps/rejected": -186.42286682128906,
      "loss": 0.6674,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.291274756193161,
      "rewards/margins": 0.0668107196688652,
      "rewards/rejected": -0.3580854535102844,
      "step": 117
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.999929691142439e-05,
      "logits/chosen": -2.062469720840454,
      "logits/rejected": -2.2004528045654297,
      "logps/chosen": -144.79664611816406,
      "logps/rejected": -149.7763671875,
      "loss": 0.6225,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.46431243419647217,
      "rewards/margins": 0.1997339278459549,
      "rewards/rejected": -0.6640464067459106,
      "step": 118
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.99992166208644e-05,
      "logits/chosen": -2.2899487018585205,
      "logits/rejected": -2.2257750034332275,
      "logps/chosen": -208.61517333984375,
      "logps/rejected": -200.343505859375,
      "loss": 0.7233,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5118069052696228,
      "rewards/margins": -0.018164699897170067,
      "rewards/rejected": -0.4936422109603882,
      "step": 119
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.999913199036719e-05,
      "logits/chosen": -2.520472288131714,
      "logits/rejected": -2.612544298171997,
      "logps/chosen": -160.17779541015625,
      "logps/rejected": -191.498291015625,
      "loss": 0.6421,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4993933439254761,
      "rewards/margins": 0.15375012159347534,
      "rewards/rejected": -0.6531434059143066,
      "step": 120
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9999043019947454e-05,
      "logits/chosen": -2.5590789318084717,
      "logits/rejected": -2.594666004180908,
      "logps/chosen": -219.09494018554688,
      "logps/rejected": -271.0511474609375,
      "loss": 0.6746,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.37969574332237244,
      "rewards/margins": 0.06747917830944061,
      "rewards/rejected": -0.44717487692832947,
      "step": 121
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9998949709620636e-05,
      "logits/chosen": -2.292649269104004,
      "logits/rejected": -2.4197192192077637,
      "logps/chosen": -156.77658081054688,
      "logps/rejected": -181.56752014160156,
      "loss": 0.6224,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24466285109519958,
      "rewards/margins": 0.17808005213737488,
      "rewards/rejected": -0.4227428734302521,
      "step": 122
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.999885205940293e-05,
      "logits/chosen": -2.526571750640869,
      "logits/rejected": -2.6078643798828125,
      "logps/chosen": -188.48431396484375,
      "logps/rejected": -209.88192749023438,
      "loss": 0.6138,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38001808524131775,
      "rewards/margins": 0.21444422006607056,
      "rewards/rejected": -0.5944622755050659,
      "step": 123
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9998750069311306e-05,
      "logits/chosen": -2.477735757827759,
      "logits/rejected": -2.524329900741577,
      "logps/chosen": -170.30780029296875,
      "logps/rejected": -167.05538940429688,
      "loss": 0.7105,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.45474332571029663,
      "rewards/margins": -0.0021874159574508667,
      "rewards/rejected": -0.45255595445632935,
      "step": 124
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.999864373936345e-05,
      "logits/chosen": -2.486284017562866,
      "logits/rejected": -2.511571168899536,
      "logps/chosen": -213.28150939941406,
      "logps/rejected": -220.744140625,
      "loss": 0.7154,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5180980563163757,
      "rewards/margins": -0.013586281798779964,
      "rewards/rejected": -0.5045117139816284,
      "step": 125
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.999853306957783e-05,
      "logits/chosen": -2.149216651916504,
      "logits/rejected": -2.331148862838745,
      "logps/chosen": -212.50167846679688,
      "logps/rejected": -229.76211547851562,
      "loss": 0.6783,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.45535096526145935,
      "rewards/margins": 0.13961246609687805,
      "rewards/rejected": -0.5949634909629822,
      "step": 126
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9998418059973654e-05,
      "logits/chosen": -2.3633456230163574,
      "logits/rejected": -2.3958113193511963,
      "logps/chosen": -260.77191162109375,
      "logps/rejected": -269.7747497558594,
      "loss": 0.624,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3385522663593292,
      "rewards/margins": 0.154096320271492,
      "rewards/rejected": -0.4926486015319824,
      "step": 127
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.99982987105709e-05,
      "logits/chosen": -2.4027421474456787,
      "logits/rejected": -2.47273588180542,
      "logps/chosen": -190.19549560546875,
      "logps/rejected": -199.79869079589844,
      "loss": 0.647,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45771288871765137,
      "rewards/margins": 0.16400352120399475,
      "rewards/rejected": -0.6217163801193237,
      "step": 128
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.999817502139027e-05,
      "logits/chosen": -2.459967851638794,
      "logits/rejected": -2.5030205249786377,
      "logps/chosen": -152.8588104248047,
      "logps/rejected": -151.28167724609375,
      "loss": 0.682,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4902128279209137,
      "rewards/margins": 0.07620880752801895,
      "rewards/rejected": -0.566421627998352,
      "step": 129
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.999804699245325e-05,
      "logits/chosen": -2.613358497619629,
      "logits/rejected": -2.6325273513793945,
      "logps/chosen": -240.51898193359375,
      "logps/rejected": -209.2674560546875,
      "loss": 0.6862,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5984245538711548,
      "rewards/margins": 0.08296503126621246,
      "rewards/rejected": -0.6813895106315613,
      "step": 130
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.999791462378206e-05,
      "logits/chosen": -2.523002862930298,
      "logits/rejected": -2.544940948486328,
      "logps/chosen": -209.53164672851562,
      "logps/rejected": -211.6482391357422,
      "loss": 0.6617,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4731237292289734,
      "rewards/margins": 0.139340341091156,
      "rewards/rejected": -0.6124641299247742,
      "step": 131
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.999777791539968e-05,
      "logits/chosen": -2.3467066287994385,
      "logits/rejected": -2.4002304077148438,
      "logps/chosen": -229.88079833984375,
      "logps/rejected": -236.3416290283203,
      "loss": 0.6545,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5287673473358154,
      "rewards/margins": 0.11849495768547058,
      "rewards/rejected": -0.6472623348236084,
      "step": 132
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9997636867329844e-05,
      "logits/chosen": -2.3951451778411865,
      "logits/rejected": -2.3899013996124268,
      "logps/chosen": -230.42471313476562,
      "logps/rejected": -204.70712280273438,
      "loss": 0.6886,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7095022797584534,
      "rewards/margins": 0.06169421225786209,
      "rewards/rejected": -0.7711963653564453,
      "step": 133
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.999749147959703e-05,
      "logits/chosen": -2.4759864807128906,
      "logits/rejected": -2.461402177810669,
      "logps/chosen": -208.62255859375,
      "logps/rejected": -215.98049926757812,
      "loss": 0.6288,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.47989076375961304,
      "rewards/margins": 0.18254442512989044,
      "rewards/rejected": -0.6624351143836975,
      "step": 134
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.99973417522265e-05,
      "logits/chosen": -2.6637802124023438,
      "logits/rejected": -2.6086812019348145,
      "logps/chosen": -201.55992126464844,
      "logps/rejected": -185.3202667236328,
      "loss": 0.7158,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5636375546455383,
      "rewards/margins": 0.09092222899198532,
      "rewards/rejected": -0.6545597910881042,
      "step": 135
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9997187685244234e-05,
      "logits/chosen": -2.386868953704834,
      "logits/rejected": -2.438833475112915,
      "logps/chosen": -149.93809509277344,
      "logps/rejected": -164.56199645996094,
      "loss": 0.7271,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.43254008889198303,
      "rewards/margins": -0.0416962131857872,
      "rewards/rejected": -0.39084386825561523,
      "step": 136
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.999702927867698e-05,
      "logits/chosen": -2.382352113723755,
      "logits/rejected": -2.447472095489502,
      "logps/chosen": -195.46499633789062,
      "logps/rejected": -191.3668212890625,
      "loss": 0.7562,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5873669385910034,
      "rewards/margins": -0.07318583130836487,
      "rewards/rejected": -0.5141811370849609,
      "step": 137
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.999686653255222e-05,
      "logits/chosen": -2.6838254928588867,
      "logits/rejected": -2.735358476638794,
      "logps/chosen": -232.84722900390625,
      "logps/rejected": -253.7335205078125,
      "loss": 0.5769,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5806477069854736,
      "rewards/margins": 0.33124226331710815,
      "rewards/rejected": -0.9118900299072266,
      "step": 138
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9996699446898235e-05,
      "logits/chosen": -2.3523752689361572,
      "logits/rejected": -2.455883026123047,
      "logps/chosen": -158.80831909179688,
      "logps/rejected": -215.85203552246094,
      "loss": 0.7447,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5600742697715759,
      "rewards/margins": 0.03419441357254982,
      "rewards/rejected": -0.5942687392234802,
      "step": 139
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.999652802174402e-05,
      "logits/chosen": -2.5024056434631348,
      "logits/rejected": -2.4992480278015137,
      "logps/chosen": -233.17120361328125,
      "logps/rejected": -249.13568115234375,
      "loss": 0.6822,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.7229613065719604,
      "rewards/margins": 0.06977183371782303,
      "rewards/rejected": -0.7927330732345581,
      "step": 140
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.999635225711933e-05,
      "logits/chosen": -2.3752949237823486,
      "logits/rejected": -2.499027729034424,
      "logps/chosen": -165.4285888671875,
      "logps/rejected": -213.1377716064453,
      "loss": 0.6571,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4794481098651886,
      "rewards/margins": 0.1226089596748352,
      "rewards/rejected": -0.602057158946991,
      "step": 141
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.999617215305468e-05,
      "logits/chosen": -2.578481674194336,
      "logits/rejected": -2.5905745029449463,
      "logps/chosen": -160.28704833984375,
      "logps/rejected": -186.60206604003906,
      "loss": 0.5771,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.379621684551239,
      "rewards/margins": 0.28185874223709106,
      "rewards/rejected": -0.6614804863929749,
      "step": 142
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.999598770958134e-05,
      "logits/chosen": -2.1814939975738525,
      "logits/rejected": -2.273460865020752,
      "logps/chosen": -178.5402069091797,
      "logps/rejected": -246.2064208984375,
      "loss": 0.6741,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.47223299741744995,
      "rewards/margins": 0.11296035349369049,
      "rewards/rejected": -0.5851933360099792,
      "step": 143
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.999579892673133e-05,
      "logits/chosen": -2.6444742679595947,
      "logits/rejected": -2.651982545852661,
      "logps/chosen": -191.74351501464844,
      "logps/rejected": -191.7657928466797,
      "loss": 0.6952,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6643162965774536,
      "rewards/margins": 0.027530232444405556,
      "rewards/rejected": -0.691846489906311,
      "step": 144
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.9995605804537426e-05,
      "logits/chosen": -2.4908974170684814,
      "logits/rejected": -2.550518274307251,
      "logps/chosen": -188.7380828857422,
      "logps/rejected": -211.574951171875,
      "loss": 0.5798,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3170633912086487,
      "rewards/margins": 0.3337726294994354,
      "rewards/rejected": -0.6508359313011169,
      "step": 145
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.999540834303315e-05,
      "logits/chosen": -2.403130054473877,
      "logits/rejected": -2.5044126510620117,
      "logps/chosen": -216.05221557617188,
      "logps/rejected": -224.658203125,
      "loss": 0.5719,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5969533920288086,
      "rewards/margins": 0.4197196662425995,
      "rewards/rejected": -1.0166730880737305,
      "step": 146
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.999520654225278e-05,
      "logits/chosen": -2.5251312255859375,
      "logits/rejected": -2.6076760292053223,
      "logps/chosen": -171.32049560546875,
      "logps/rejected": -189.77108764648438,
      "loss": 0.5929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5219372510910034,
      "rewards/margins": 0.3483983874320984,
      "rewards/rejected": -0.8703356385231018,
      "step": 147
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.9995000402231354e-05,
      "logits/chosen": -2.564474105834961,
      "logits/rejected": -2.510350227355957,
      "logps/chosen": -219.81732177734375,
      "logps/rejected": -202.83493041992188,
      "loss": 0.7793,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7294387817382812,
      "rewards/margins": -0.11600115895271301,
      "rewards/rejected": -0.6134375333786011,
      "step": 148
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.999478992300466e-05,
      "logits/chosen": -2.5275182723999023,
      "logits/rejected": -2.6380527019500732,
      "logps/chosen": -165.94358825683594,
      "logps/rejected": -212.28086853027344,
      "loss": 0.565,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4685615599155426,
      "rewards/margins": 0.44902655482292175,
      "rewards/rejected": -0.9175881147384644,
      "step": 149
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.999457510460923e-05,
      "logits/chosen": -2.407663106918335,
      "logits/rejected": -2.5699005126953125,
      "logps/chosen": -167.9168701171875,
      "logps/rejected": -214.4752197265625,
      "loss": 0.5801,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.48059117794036865,
      "rewards/margins": 0.3546959161758423,
      "rewards/rejected": -0.8352870345115662,
      "step": 150
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.999435594708236e-05,
      "logits/chosen": -2.3764097690582275,
      "logits/rejected": -2.3675248622894287,
      "logps/chosen": -201.25814819335938,
      "logps/rejected": -229.6868133544922,
      "loss": 0.6714,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6616516709327698,
      "rewards/margins": 0.10846880823373795,
      "rewards/rejected": -0.7701205611228943,
      "step": 151
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.999413245046211e-05,
      "logits/chosen": -2.5667271614074707,
      "logits/rejected": -2.5309338569641113,
      "logps/chosen": -210.30267333984375,
      "logps/rejected": -206.71014404296875,
      "loss": 0.5848,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.572422444820404,
      "rewards/margins": 0.30542662739753723,
      "rewards/rejected": -0.8778490424156189,
      "step": 152
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.9993904614787254e-05,
      "logits/chosen": -2.463524341583252,
      "logits/rejected": -2.530627965927124,
      "logps/chosen": -179.8057861328125,
      "logps/rejected": -200.56394958496094,
      "loss": 0.6231,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6925724148750305,
      "rewards/margins": 0.21750670671463013,
      "rewards/rejected": -0.9100791215896606,
      "step": 153
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.999367244009736e-05,
      "logits/chosen": -2.4720635414123535,
      "logits/rejected": -2.5370302200317383,
      "logps/chosen": -194.29440307617188,
      "logps/rejected": -196.02944946289062,
      "loss": 0.8265,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -1.0657517910003662,
      "rewards/margins": -0.16590073704719543,
      "rewards/rejected": -0.8998512029647827,
      "step": 154
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.999343592643274e-05,
      "logits/chosen": -2.3978195190429688,
      "logits/rejected": -2.397984266281128,
      "logps/chosen": -229.4518585205078,
      "logps/rejected": -217.56869506835938,
      "loss": 0.6598,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6397801637649536,
      "rewards/margins": 0.10495337843894958,
      "rewards/rejected": -0.7447335720062256,
      "step": 155
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.999319507383444e-05,
      "logits/chosen": -2.3354411125183105,
      "logits/rejected": -2.4454195499420166,
      "logps/chosen": -149.3455810546875,
      "logps/rejected": -187.3743438720703,
      "loss": 0.5981,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7036041617393494,
      "rewards/margins": 0.3835078775882721,
      "rewards/rejected": -1.0871120691299438,
      "step": 156
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.999294988234428e-05,
      "logits/chosen": -2.491671323776245,
      "logits/rejected": -2.550612449645996,
      "logps/chosen": -180.36758422851562,
      "logps/rejected": -251.5354461669922,
      "loss": 0.6038,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7723766565322876,
      "rewards/margins": 0.2752906382083893,
      "rewards/rejected": -1.047667384147644,
      "step": 157
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.999270035200483e-05,
      "logits/chosen": -2.492790937423706,
      "logits/rejected": -2.365570306777954,
      "logps/chosen": -195.1597900390625,
      "logps/rejected": -185.96673583984375,
      "loss": 0.7169,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.2230509519577026,
      "rewards/margins": 0.14210942387580872,
      "rewards/rejected": -1.3651604652404785,
      "step": 158
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.99924464828594e-05,
      "logits/chosen": -2.412907838821411,
      "logits/rejected": -2.508164882659912,
      "logps/chosen": -196.51846313476562,
      "logps/rejected": -218.0734405517578,
      "loss": 0.5942,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.813502848148346,
      "rewards/margins": 0.3059760332107544,
      "rewards/rejected": -1.1194789409637451,
      "step": 159
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.9992188274952064e-05,
      "logits/chosen": -2.4737277030944824,
      "logits/rejected": -2.5720887184143066,
      "logps/chosen": -155.16171264648438,
      "logps/rejected": -195.86997985839844,
      "loss": 0.7379,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.8073078989982605,
      "rewards/margins": 0.11351939290761948,
      "rewards/rejected": -0.920827329158783,
      "step": 160
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.999192572832765e-05,
      "logits/chosen": -2.5303239822387695,
      "logits/rejected": -2.5628015995025635,
      "logps/chosen": -281.7694396972656,
      "logps/rejected": -303.6089782714844,
      "loss": 0.6735,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6562893986701965,
      "rewards/margins": 0.12363787740468979,
      "rewards/rejected": -0.7799273133277893,
      "step": 161
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.999165884303174e-05,
      "logits/chosen": -2.481058359146118,
      "logits/rejected": -2.4526753425598145,
      "logps/chosen": -182.2716064453125,
      "logps/rejected": -242.58828735351562,
      "loss": 0.7317,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.7687569260597229,
      "rewards/margins": -0.007903970777988434,
      "rewards/rejected": -0.7608529329299927,
      "step": 162
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.999138761911066e-05,
      "logits/chosen": -2.459987163543701,
      "logits/rejected": -2.5180883407592773,
      "logps/chosen": -209.83102416992188,
      "logps/rejected": -216.1147918701172,
      "loss": 0.7265,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.0292317867279053,
      "rewards/margins": 0.04909829795360565,
      "rewards/rejected": -1.0783300399780273,
      "step": 163
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.99911120566115e-05,
      "logits/chosen": -2.5340957641601562,
      "logits/rejected": -2.654463052749634,
      "logps/chosen": -210.88929748535156,
      "logps/rejected": -198.64614868164062,
      "loss": 0.7475,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.2450902462005615,
      "rewards/margins": 0.012418277561664581,
      "rewards/rejected": -1.2575085163116455,
      "step": 164
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.99908321555821e-05,
      "logits/chosen": -2.4791364669799805,
      "logits/rejected": -2.5890302658081055,
      "logps/chosen": -192.5817108154297,
      "logps/rejected": -224.9603271484375,
      "loss": 0.6089,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.9010199308395386,
      "rewards/margins": 0.4015497863292694,
      "rewards/rejected": -1.3025696277618408,
      "step": 165
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.999054791607105e-05,
      "logits/chosen": -2.428842067718506,
      "logits/rejected": -2.372598171234131,
      "logps/chosen": -221.89833068847656,
      "logps/rejected": -212.29595947265625,
      "loss": 0.7305,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.0515369176864624,
      "rewards/margins": 0.017772257328033447,
      "rewards/rejected": -1.0693092346191406,
      "step": 166
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.999025933812769e-05,
      "logits/chosen": -2.50471830368042,
      "logits/rejected": -2.5308945178985596,
      "logps/chosen": -198.7537384033203,
      "logps/rejected": -236.5478515625,
      "loss": 0.6487,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8205583095550537,
      "rewards/margins": 0.18831993639469147,
      "rewards/rejected": -1.008878231048584,
      "step": 167
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.9989966421802114e-05,
      "logits/chosen": -2.410905122756958,
      "logits/rejected": -2.5401611328125,
      "logps/chosen": -202.42991638183594,
      "logps/rejected": -219.6848602294922,
      "loss": 0.6257,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9180124402046204,
      "rewards/margins": 0.24881067872047424,
      "rewards/rejected": -1.166823148727417,
      "step": 168
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.998966916714519e-05,
      "logits/chosen": -2.4587323665618896,
      "logits/rejected": -2.5724194049835205,
      "logps/chosen": -154.88470458984375,
      "logps/rejected": -172.47268676757812,
      "loss": 0.618,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.7627564072608948,
      "rewards/margins": 0.294120192527771,
      "rewards/rejected": -1.0568766593933105,
      "step": 169
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.998936757420851e-05,
      "logits/chosen": -2.284227132797241,
      "logits/rejected": -2.380892515182495,
      "logps/chosen": -138.282958984375,
      "logps/rejected": -180.36441040039062,
      "loss": 0.6089,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7448742389678955,
      "rewards/margins": 0.4253446161746979,
      "rewards/rejected": -1.1702189445495605,
      "step": 170
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.9989061643044434e-05,
      "logits/chosen": -2.5783276557922363,
      "logits/rejected": -2.5471560955047607,
      "logps/chosen": -186.851318359375,
      "logps/rejected": -193.73837280273438,
      "loss": 0.7108,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0091472864151,
      "rewards/margins": 0.033400412648916245,
      "rewards/rejected": -1.0425477027893066,
      "step": 171
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.9988751373706075e-05,
      "logits/chosen": -2.398940086364746,
      "logits/rejected": -2.3498027324676514,
      "logps/chosen": -171.69107055664062,
      "logps/rejected": -209.19424438476562,
      "loss": 0.7045,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9662235975265503,
      "rewards/margins": 0.137020543217659,
      "rewards/rejected": -1.1032441854476929,
      "step": 172
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.9988436766247284e-05,
      "logits/chosen": -2.415847063064575,
      "logits/rejected": -2.3855319023132324,
      "logps/chosen": -228.1761932373047,
      "logps/rejected": -195.01991271972656,
      "loss": 0.6696,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9869641661643982,
      "rewards/margins": 0.11401443928480148,
      "rewards/rejected": -1.1009787321090698,
      "step": 173
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.9988117820722704e-05,
      "logits/chosen": -2.638780117034912,
      "logits/rejected": -2.607104539871216,
      "logps/chosen": -234.91183471679688,
      "logps/rejected": -203.67996215820312,
      "loss": 0.9279,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.2375922203063965,
      "rewards/margins": -0.281325101852417,
      "rewards/rejected": -0.956267237663269,
      "step": 174
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.998779453718768e-05,
      "logits/chosen": -2.254124879837036,
      "logits/rejected": -2.429471969604492,
      "logps/chosen": -235.76698303222656,
      "logps/rejected": -262.7959899902344,
      "loss": 0.6238,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.0454847812652588,
      "rewards/margins": 0.3030804395675659,
      "rewards/rejected": -1.3485652208328247,
      "step": 175
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.9987466915698346e-05,
      "logits/chosen": -2.276986598968506,
      "logits/rejected": -2.337780475616455,
      "logps/chosen": -138.3777313232422,
      "logps/rejected": -166.30181884765625,
      "loss": 0.5682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5320202708244324,
      "rewards/margins": 0.315082311630249,
      "rewards/rejected": -0.8471025228500366,
      "step": 176
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.998713495631156e-05,
      "logits/chosen": -2.420923948287964,
      "logits/rejected": -2.565662384033203,
      "logps/chosen": -180.71446228027344,
      "logps/rejected": -192.2035369873047,
      "loss": 0.647,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8985862731933594,
      "rewards/margins": 0.2061445415019989,
      "rewards/rejected": -1.1047309637069702,
      "step": 177
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.998679865908499e-05,
      "logits/chosen": -2.5881500244140625,
      "logits/rejected": -2.6503522396087646,
      "logps/chosen": -183.07540893554688,
      "logps/rejected": -233.943603515625,
      "loss": 0.6889,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.1374621391296387,
      "rewards/margins": 0.19141921401023865,
      "rewards/rejected": -1.3288813829421997,
      "step": 178
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.9986458024076984e-05,
      "logits/chosen": -2.4952712059020996,
      "logits/rejected": -2.50299072265625,
      "logps/chosen": -192.31314086914062,
      "logps/rejected": -197.241455078125,
      "loss": 0.7729,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -1.13700532913208,
      "rewards/margins": -0.10435198992490768,
      "rewards/rejected": -1.0326533317565918,
      "step": 179
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.998611305134669e-05,
      "logits/chosen": -2.5952248573303223,
      "logits/rejected": -2.600719928741455,
      "logps/chosen": -191.4530487060547,
      "logps/rejected": -212.82955932617188,
      "loss": 0.674,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9059873819351196,
      "rewards/margins": 0.1599172055721283,
      "rewards/rejected": -1.0659046173095703,
      "step": 180
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.9985763740954e-05,
      "logits/chosen": -2.4821391105651855,
      "logits/rejected": -2.4574878215789795,
      "logps/chosen": -264.071533203125,
      "logps/rejected": -256.51611328125,
      "loss": 0.7333,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.0796548128128052,
      "rewards/margins": 0.06612614542245865,
      "rewards/rejected": -1.1457810401916504,
      "step": 181
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.9985410092959553e-05,
      "logits/chosen": -2.4601786136627197,
      "logits/rejected": -2.519011974334717,
      "logps/chosen": -158.57687377929688,
      "logps/rejected": -173.29666137695312,
      "loss": 0.7693,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.7817510366439819,
      "rewards/margins": -0.05254516005516052,
      "rewards/rejected": -0.729205846786499,
      "step": 182
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.998505210742472e-05,
      "logits/chosen": -2.6099700927734375,
      "logits/rejected": -2.58197021484375,
      "logps/chosen": -195.2617950439453,
      "logps/rejected": -188.54165649414062,
      "loss": 0.9197,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -1.1366631984710693,
      "rewards/margins": -0.32979804277420044,
      "rewards/rejected": -0.8068650960922241,
      "step": 183
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.9984689784411686e-05,
      "logits/chosen": -2.679287910461426,
      "logits/rejected": -2.5679209232330322,
      "logps/chosen": -211.75323486328125,
      "logps/rejected": -245.02603149414062,
      "loss": 0.6702,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9691303372383118,
      "rewards/margins": 0.17891569435596466,
      "rewards/rejected": -1.1480460166931152,
      "step": 184
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.9984323123983334e-05,
      "logits/chosen": -2.537733316421509,
      "logits/rejected": -2.6055424213409424,
      "logps/chosen": -151.74288940429688,
      "logps/rejected": -250.8179168701172,
      "loss": 0.6366,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.928146243095398,
      "rewards/margins": 0.24633805453777313,
      "rewards/rejected": -1.174484372138977,
      "step": 185
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.998395212620332e-05,
      "logits/chosen": -2.4368271827697754,
      "logits/rejected": -2.529536247253418,
      "logps/chosen": -207.87677001953125,
      "logps/rejected": -229.0767822265625,
      "loss": 0.6199,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.7506311535835266,
      "rewards/margins": 0.24917976558208466,
      "rewards/rejected": -0.9998109340667725,
      "step": 186
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.998357679113603e-05,
      "logits/chosen": -2.5217807292938232,
      "logits/rejected": -2.6954245567321777,
      "logps/chosen": -192.20407104492188,
      "logps/rejected": -200.1387176513672,
      "loss": 0.6013,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6829215288162231,
      "rewards/margins": 0.2588457763195038,
      "rewards/rejected": -0.9417673349380493,
      "step": 187
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.9983197118846655e-05,
      "logits/chosen": -2.4711546897888184,
      "logits/rejected": -2.4839439392089844,
      "logps/chosen": -160.9691925048828,
      "logps/rejected": -167.21469116210938,
      "loss": 0.7301,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7170782685279846,
      "rewards/margins": 0.04331670328974724,
      "rewards/rejected": -0.7603949904441833,
      "step": 188
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.9982813109401096e-05,
      "logits/chosen": -2.3429412841796875,
      "logits/rejected": -2.3843770027160645,
      "logps/chosen": -221.22274780273438,
      "logps/rejected": -246.99005126953125,
      "loss": 0.7711,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.114179015159607,
      "rewards/margins": -0.07230883836746216,
      "rewards/rejected": -1.0418701171875,
      "step": 189
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.998242476286601e-05,
      "logits/chosen": -2.7864720821380615,
      "logits/rejected": -2.7745487689971924,
      "logps/chosen": -194.08201599121094,
      "logps/rejected": -226.28591918945312,
      "loss": 0.71,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.8893502354621887,
      "rewards/margins": 0.06053268164396286,
      "rewards/rejected": -0.949882984161377,
      "step": 190
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.998203207930882e-05,
      "logits/chosen": -2.6679065227508545,
      "logits/rejected": -2.711076498031616,
      "logps/chosen": -190.53013610839844,
      "logps/rejected": -200.187744140625,
      "loss": 0.8201,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -1.0141409635543823,
      "rewards/margins": -0.17862781882286072,
      "rewards/rejected": -0.8355131149291992,
      "step": 191
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.998163505879769e-05,
      "logits/chosen": -2.7404122352600098,
      "logits/rejected": -2.732025146484375,
      "logps/chosen": -178.48135375976562,
      "logps/rejected": -183.08074951171875,
      "loss": 0.7078,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8886659145355225,
      "rewards/margins": 0.043551601469516754,
      "rewards/rejected": -0.9322174787521362,
      "step": 192
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.998123370140156e-05,
      "logits/chosen": -2.6989643573760986,
      "logits/rejected": -2.7124342918395996,
      "logps/chosen": -223.03314208984375,
      "logps/rejected": -221.12120056152344,
      "loss": 0.6559,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.764815628528595,
      "rewards/margins": 0.1835068166255951,
      "rewards/rejected": -0.9483224153518677,
      "step": 193
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.99808280071901e-05,
      "logits/chosen": -2.618727684020996,
      "logits/rejected": -2.6078970432281494,
      "logps/chosen": -176.1873779296875,
      "logps/rejected": -199.52859497070312,
      "loss": 0.7406,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.8613646626472473,
      "rewards/margins": -0.05548467859625816,
      "rewards/rejected": -0.805880069732666,
      "step": 194
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.9980417976233735e-05,
      "logits/chosen": -2.6192498207092285,
      "logits/rejected": -2.732285261154175,
      "logps/chosen": -211.3184814453125,
      "logps/rejected": -195.73056030273438,
      "loss": 0.7921,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.9030998945236206,
      "rewards/margins": -0.1349114179611206,
      "rewards/rejected": -0.7681884765625,
      "step": 195
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.9980003608603656e-05,
      "logits/chosen": -2.5657694339752197,
      "logits/rejected": -2.5230712890625,
      "logps/chosen": -203.0819091796875,
      "logps/rejected": -226.5617218017578,
      "loss": 0.6618,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.609666109085083,
      "rewards/margins": 0.08627453446388245,
      "rewards/rejected": -0.6959406137466431,
      "step": 196
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.997958490437178e-05,
      "logits/chosen": -2.641559362411499,
      "logits/rejected": -2.76076602935791,
      "logps/chosen": -169.1241455078125,
      "logps/rejected": -183.4690399169922,
      "loss": 0.7612,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7050959467887878,
      "rewards/margins": -0.03217136114835739,
      "rewards/rejected": -0.6729245781898499,
      "step": 197
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.9979161863610816e-05,
      "logits/chosen": -2.441647529602051,
      "logits/rejected": -2.467510223388672,
      "logps/chosen": -143.13748168945312,
      "logps/rejected": -172.0884246826172,
      "loss": 0.6647,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5210601091384888,
      "rewards/margins": 0.14004816114902496,
      "rewards/rejected": -0.6611082553863525,
      "step": 198
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.99787344863942e-05,
      "logits/chosen": -2.627023458480835,
      "logits/rejected": -2.5984585285186768,
      "logps/chosen": -203.31980895996094,
      "logps/rejected": -218.84060668945312,
      "loss": 0.6172,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5382375717163086,
      "rewards/margins": 0.20881405472755432,
      "rewards/rejected": -0.7470515966415405,
      "step": 199
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.997830277279612e-05,
      "logits/chosen": -2.595003604888916,
      "logits/rejected": -2.722808361053467,
      "logps/chosen": -223.01495361328125,
      "logps/rejected": -240.90130615234375,
      "loss": 0.7333,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.7480457425117493,
      "rewards/margins": -0.029575012624263763,
      "rewards/rejected": -0.7184706926345825,
      "step": 200
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.997786672289152e-05,
      "logits/chosen": -2.6220602989196777,
      "logits/rejected": -2.6606202125549316,
      "logps/chosen": -204.98797607421875,
      "logps/rejected": -206.25839233398438,
      "loss": 0.7368,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6979295611381531,
      "rewards/margins": -0.07002773880958557,
      "rewards/rejected": -0.6279018521308899,
      "step": 201
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.997742633675612e-05,
      "logits/chosen": -2.547811508178711,
      "logits/rejected": -2.551816463470459,
      "logps/chosen": -196.38748168945312,
      "logps/rejected": -222.4247589111328,
      "loss": 0.6402,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5409201383590698,
      "rewards/margins": 0.19667872786521912,
      "rewards/rejected": -0.7375988364219666,
      "step": 202
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.9976981614466344e-05,
      "logits/chosen": -2.2461767196655273,
      "logits/rejected": -2.2968175411224365,
      "logps/chosen": -185.43954467773438,
      "logps/rejected": -196.11392211914062,
      "loss": 0.7743,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5852402448654175,
      "rewards/margins": -0.08380623161792755,
      "rewards/rejected": -0.5014340281486511,
      "step": 203
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.997653255609942e-05,
      "logits/chosen": -2.4160966873168945,
      "logits/rejected": -2.389375925064087,
      "logps/chosen": -189.16510009765625,
      "logps/rejected": -202.54649353027344,
      "loss": 0.7306,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.729293167591095,
      "rewards/margins": -0.006243497133255005,
      "rewards/rejected": -0.7230496406555176,
      "step": 204
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.997607916173329e-05,
      "logits/chosen": -2.6787378787994385,
      "logits/rejected": -2.6836445331573486,
      "logps/chosen": -177.60980224609375,
      "logps/rejected": -177.10487365722656,
      "loss": 0.765,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5183064341545105,
      "rewards/margins": -0.11038734763860703,
      "rewards/rejected": -0.4079190790653229,
      "step": 205
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.997562143144668e-05,
      "logits/chosen": -2.559563398361206,
      "logits/rejected": -2.5205612182617188,
      "logps/chosen": -208.58892822265625,
      "logps/rejected": -248.25648498535156,
      "loss": 0.6524,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5725008845329285,
      "rewards/margins": 0.11152593046426773,
      "rewards/rejected": -0.6840267777442932,
      "step": 206
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.997515936531903e-05,
      "logits/chosen": -2.6034438610076904,
      "logits/rejected": -2.528724193572998,
      "logps/chosen": -159.13522338867188,
      "logps/rejected": -135.77056884765625,
      "loss": 0.7413,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5185827016830444,
      "rewards/margins": -0.04235066473484039,
      "rewards/rejected": -0.47623202204704285,
      "step": 207
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.9974692963430595e-05,
      "logits/chosen": -2.3184618949890137,
      "logits/rejected": -2.3163304328918457,
      "logps/chosen": -130.70643615722656,
      "logps/rejected": -142.0390625,
      "loss": 0.7362,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6057229042053223,
      "rewards/margins": -0.014669769443571568,
      "rewards/rejected": -0.5910531282424927,
      "step": 208
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.99742222258623e-05,
      "logits/chosen": -2.660839796066284,
      "logits/rejected": -2.6470634937286377,
      "logps/chosen": -221.08424377441406,
      "logps/rejected": -213.25003051757812,
      "loss": 0.6922,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.5536531209945679,
      "rewards/margins": 0.022671688348054886,
      "rewards/rejected": -0.5763247609138489,
      "step": 209
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.997374715269589e-05,
      "logits/chosen": -2.2223334312438965,
      "logits/rejected": -2.2852203845977783,
      "logps/chosen": -172.71421813964844,
      "logps/rejected": -206.3236541748047,
      "loss": 0.6722,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5476804375648499,
      "rewards/margins": 0.10758630931377411,
      "rewards/rejected": -0.6552667617797852,
      "step": 210
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.997326774401383e-05,
      "logits/chosen": -2.262892961502075,
      "logits/rejected": -2.343332052230835,
      "logps/chosen": -152.0110626220703,
      "logps/rejected": -178.68345642089844,
      "loss": 0.694,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48357081413269043,
      "rewards/margins": 0.07582694292068481,
      "rewards/rejected": -0.55939781665802,
      "step": 211
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.9972783999899366e-05,
      "logits/chosen": -2.2378625869750977,
      "logits/rejected": -2.2833354473114014,
      "logps/chosen": -154.71405029296875,
      "logps/rejected": -177.74502563476562,
      "loss": 0.709,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.5034060478210449,
      "rewards/margins": -0.0023919548839330673,
      "rewards/rejected": -0.5010141134262085,
      "step": 212
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.997229592043645e-05,
      "logits/chosen": -2.4182140827178955,
      "logits/rejected": -2.477410078048706,
      "logps/chosen": -217.6938018798828,
      "logps/rejected": -236.60140991210938,
      "loss": 0.7474,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.42065930366516113,
      "rewards/margins": -0.08166539669036865,
      "rewards/rejected": -0.33899393677711487,
      "step": 213
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.997180350570984e-05,
      "logits/chosen": -2.6971006393432617,
      "logits/rejected": -2.709580659866333,
      "logps/chosen": -231.80714416503906,
      "logps/rejected": -234.41290283203125,
      "loss": 0.6593,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4388115406036377,
      "rewards/margins": 0.1023733913898468,
      "rewards/rejected": -0.5411849617958069,
      "step": 214
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.9971306755804995e-05,
      "logits/chosen": -2.600799083709717,
      "logits/rejected": -2.5639290809631348,
      "logps/chosen": -211.75051879882812,
      "logps/rejected": -185.4350128173828,
      "loss": 0.6654,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3958936929702759,
      "rewards/margins": 0.08459535241127014,
      "rewards/rejected": -0.4804890751838684,
      "step": 215
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.997080567080817e-05,
      "logits/chosen": -2.2965052127838135,
      "logits/rejected": -2.3450510501861572,
      "logps/chosen": -170.9476318359375,
      "logps/rejected": -174.3068389892578,
      "loss": 0.6604,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5262787938117981,
      "rewards/margins": 0.09660908579826355,
      "rewards/rejected": -0.6228878498077393,
      "step": 216
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.9970300250806346e-05,
      "logits/chosen": -2.5405097007751465,
      "logits/rejected": -2.5496726036071777,
      "logps/chosen": -266.0992736816406,
      "logps/rejected": -260.38275146484375,
      "loss": 0.7048,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5492247939109802,
      "rewards/margins": 0.042401984333992004,
      "rewards/rejected": -0.591626763343811,
      "step": 217
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.996979049588727e-05,
      "logits/chosen": -2.502474308013916,
      "logits/rejected": -2.439131736755371,
      "logps/chosen": -227.49476623535156,
      "logps/rejected": -206.38299560546875,
      "loss": 0.711,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5280985236167908,
      "rewards/margins": 0.010388553142547607,
      "rewards/rejected": -0.5384870767593384,
      "step": 218
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.996927640613944e-05,
      "logits/chosen": -2.3517448902130127,
      "logits/rejected": -2.5404696464538574,
      "logps/chosen": -162.60267639160156,
      "logps/rejected": -216.92562866210938,
      "loss": 0.6095,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2759331464767456,
      "rewards/margins": 0.19823111593723297,
      "rewards/rejected": -0.474164217710495,
      "step": 219
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.99687579816521e-05,
      "logits/chosen": -2.411978006362915,
      "logits/rejected": -2.5019097328186035,
      "logps/chosen": -201.91445922851562,
      "logps/rejected": -211.31448364257812,
      "loss": 0.6797,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4115257263183594,
      "rewards/margins": 0.07280793786048889,
      "rewards/rejected": -0.48433366417884827,
      "step": 220
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.9968235222515246e-05,
      "logits/chosen": -2.4315428733825684,
      "logits/rejected": -2.4550862312316895,
      "logps/chosen": -162.66555786132812,
      "logps/rejected": -156.19947814941406,
      "loss": 0.6482,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.36304140090942383,
      "rewards/margins": 0.12375527620315552,
      "rewards/rejected": -0.48679661750793457,
      "step": 221
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.996770812881964e-05,
      "logits/chosen": -2.5614113807678223,
      "logits/rejected": -2.5391292572021484,
      "logps/chosen": -214.3634033203125,
      "logps/rejected": -209.9477996826172,
      "loss": 0.7018,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.319223552942276,
      "rewards/margins": 0.018056731671094894,
      "rewards/rejected": -0.3372803032398224,
      "step": 222
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.9967176700656776e-05,
      "logits/chosen": -2.438467264175415,
      "logits/rejected": -2.4638776779174805,
      "logps/chosen": -227.1153564453125,
      "logps/rejected": -219.0038299560547,
      "loss": 0.6209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45765751600265503,
      "rewards/margins": 0.19506625831127167,
      "rewards/rejected": -0.6527237892150879,
      "step": 223
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.996664093811892e-05,
      "logits/chosen": -2.5968563556671143,
      "logits/rejected": -2.5226142406463623,
      "logps/chosen": -184.9003143310547,
      "logps/rejected": -202.8119354248047,
      "loss": 0.5973,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.36458614468574524,
      "rewards/margins": 0.22884023189544678,
      "rewards/rejected": -0.5934264063835144,
      "step": 224
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.996610084129908e-05,
      "logits/chosen": -2.356076955795288,
      "logits/rejected": -2.464617967605591,
      "logps/chosen": -233.56741333007812,
      "logps/rejected": -252.82614135742188,
      "loss": 0.7174,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.475138783454895,
      "rewards/margins": 0.015569500625133514,
      "rewards/rejected": -0.49070829153060913,
      "step": 225
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.996555641029101e-05,
      "logits/chosen": -2.4807751178741455,
      "logits/rejected": -2.5052688121795654,
      "logps/chosen": -218.75250244140625,
      "logps/rejected": -202.75149536132812,
      "loss": 0.7541,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5363876819610596,
      "rewards/margins": -0.06761842221021652,
      "rewards/rejected": -0.4687691926956177,
      "step": 226
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.996500764518923e-05,
      "logits/chosen": -2.5997631549835205,
      "logits/rejected": -2.6119892597198486,
      "logps/chosen": -212.090087890625,
      "logps/rejected": -220.1112823486328,
      "loss": 0.7185,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.5663864016532898,
      "rewards/margins": -0.016858436167240143,
      "rewards/rejected": -0.5495280027389526,
      "step": 227
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.9964454546089026e-05,
      "logits/chosen": -2.25441837310791,
      "logits/rejected": -2.251708507537842,
      "logps/chosen": -216.14996337890625,
      "logps/rejected": -218.1949005126953,
      "loss": 0.7617,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.46758705377578735,
      "rewards/margins": -0.09628176689147949,
      "rewards/rejected": -0.37130531668663025,
      "step": 228
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.996389711308639e-05,
      "logits/chosen": -2.475829601287842,
      "logits/rejected": -2.4804282188415527,
      "logps/chosen": -197.95965576171875,
      "logps/rejected": -227.29605102539062,
      "loss": 0.6275,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2889898717403412,
      "rewards/margins": 0.18446122109889984,
      "rewards/rejected": -0.47345107793807983,
      "step": 229
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.996333534627809e-05,
      "logits/chosen": -2.4503121376037598,
      "logits/rejected": -2.40040922164917,
      "logps/chosen": -190.1997833251953,
      "logps/rejected": -170.68679809570312,
      "loss": 0.7168,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4417100250720978,
      "rewards/margins": -0.0062090009450912476,
      "rewards/rejected": -0.4355010688304901,
      "step": 230
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.996276924576169e-05,
      "logits/chosen": -2.5104312896728516,
      "logits/rejected": -2.5510809421539307,
      "logps/chosen": -134.1800079345703,
      "logps/rejected": -153.13365173339844,
      "loss": 0.7468,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.42858752608299255,
      "rewards/margins": -0.08554935455322266,
      "rewards/rejected": -0.3430381715297699,
      "step": 231
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.996219881163543e-05,
      "logits/chosen": -2.456216335296631,
      "logits/rejected": -2.4285402297973633,
      "logps/chosen": -209.37985229492188,
      "logps/rejected": -195.06568908691406,
      "loss": 0.7159,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.46927523612976074,
      "rewards/margins": -0.0166518222540617,
      "rewards/rejected": -0.4526233971118927,
      "step": 232
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.996162404399835e-05,
      "logits/chosen": -2.450845956802368,
      "logits/rejected": -2.4209864139556885,
      "logps/chosen": -198.9377899169922,
      "logps/rejected": -203.7089385986328,
      "loss": 0.6708,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4800047278404236,
      "rewards/margins": 0.09784451127052307,
      "rewards/rejected": -0.5778492093086243,
      "step": 233
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.996104494295024e-05,
      "logits/chosen": -2.5744130611419678,
      "logits/rejected": -2.641494035720825,
      "logps/chosen": -182.59878540039062,
      "logps/rejected": -207.31797790527344,
      "loss": 0.6749,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4756154417991638,
      "rewards/margins": 0.08592241257429123,
      "rewards/rejected": -0.5615378618240356,
      "step": 234
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.996046150859161e-05,
      "logits/chosen": -2.0956687927246094,
      "logits/rejected": -2.1018872261047363,
      "logps/chosen": -200.86215209960938,
      "logps/rejected": -221.92138671875,
      "loss": 0.6445,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.488430380821228,
      "rewards/margins": 0.1436658799648285,
      "rewards/rejected": -0.6320962905883789,
      "step": 235
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.9959873741023774e-05,
      "logits/chosen": -2.3680927753448486,
      "logits/rejected": -2.415205240249634,
      "logps/chosen": -163.829833984375,
      "logps/rejected": -182.74481201171875,
      "loss": 0.6839,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.35495901107788086,
      "rewards/margins": 0.03828255832195282,
      "rewards/rejected": -0.3932415544986725,
      "step": 236
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.995928164034876e-05,
      "logits/chosen": -2.5983476638793945,
      "logits/rejected": -2.7171671390533447,
      "logps/chosen": -189.3108673095703,
      "logps/rejected": -195.94888305664062,
      "loss": 0.6611,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4241011142730713,
      "rewards/margins": 0.0877021998167038,
      "rewards/rejected": -0.5118032693862915,
      "step": 237
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.995868520666936e-05,
      "logits/chosen": -2.510660171508789,
      "logits/rejected": -2.5224432945251465,
      "logps/chosen": -227.51687622070312,
      "logps/rejected": -231.05897521972656,
      "loss": 0.7269,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.40695255994796753,
      "rewards/margins": -0.046068765223026276,
      "rewards/rejected": -0.36088380217552185,
      "step": 238
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.9958084440089095e-05,
      "logits/chosen": -2.256051540374756,
      "logits/rejected": -2.2608485221862793,
      "logps/chosen": -209.19252014160156,
      "logps/rejected": -206.2628631591797,
      "loss": 0.6338,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3002999424934387,
      "rewards/margins": 0.18500341475009918,
      "rewards/rejected": -0.4853033423423767,
      "step": 239
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.995747934071229e-05,
      "logits/chosen": -2.8373260498046875,
      "logits/rejected": -2.8713366985321045,
      "logps/chosen": -186.55723571777344,
      "logps/rejected": -221.16917419433594,
      "loss": 0.6489,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3408931493759155,
      "rewards/margins": 0.18887601792812347,
      "rewards/rejected": -0.5297691822052002,
      "step": 240
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.995686990864398e-05,
      "logits/chosen": -2.328822135925293,
      "logits/rejected": -2.4501209259033203,
      "logps/chosen": -164.64404296875,
      "logps/rejected": -164.53396606445312,
      "loss": 0.6872,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5059252977371216,
      "rewards/margins": 0.046310462057590485,
      "rewards/rejected": -0.5522357225418091,
      "step": 241
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.995625614398996e-05,
      "logits/chosen": -2.254148006439209,
      "logits/rejected": -2.45564603805542,
      "logps/chosen": -215.61080932617188,
      "logps/rejected": -236.83526611328125,
      "loss": 0.618,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3230094313621521,
      "rewards/margins": 0.2154739797115326,
      "rewards/rejected": -0.5384833812713623,
      "step": 242
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.995563804685678e-05,
      "logits/chosen": -2.4298973083496094,
      "logits/rejected": -2.6152379512786865,
      "logps/chosen": -171.45925903320312,
      "logps/rejected": -215.27342224121094,
      "loss": 0.6461,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5956869721412659,
      "rewards/margins": 0.1696903109550476,
      "rewards/rejected": -0.7653772830963135,
      "step": 243
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.995501561735176e-05,
      "logits/chosen": -2.5064899921417236,
      "logits/rejected": -2.667478561401367,
      "logps/chosen": -201.28167724609375,
      "logps/rejected": -200.6697998046875,
      "loss": 0.7252,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.50804603099823,
      "rewards/margins": -0.021207518875598907,
      "rewards/rejected": -0.48683851957321167,
      "step": 244
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.995438885558294e-05,
      "logits/chosen": -2.575934886932373,
      "logits/rejected": -2.5915310382843018,
      "logps/chosen": -241.2596893310547,
      "logps/rejected": -236.16864013671875,
      "loss": 0.6113,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.41618824005126953,
      "rewards/margins": 0.21841469407081604,
      "rewards/rejected": -0.6346028447151184,
      "step": 245
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.995375776165913e-05,
      "logits/chosen": -2.6145882606506348,
      "logits/rejected": -2.6391665935516357,
      "logps/chosen": -230.53524780273438,
      "logps/rejected": -228.5028076171875,
      "loss": 0.7431,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5892909169197083,
      "rewards/margins": -0.044729601591825485,
      "rewards/rejected": -0.5445613265037537,
      "step": 246
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.995312233568989e-05,
      "logits/chosen": -2.279015302658081,
      "logits/rejected": -2.4627904891967773,
      "logps/chosen": -205.08226013183594,
      "logps/rejected": -245.180908203125,
      "loss": 0.6492,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.37771591544151306,
      "rewards/margins": 0.17377406358718872,
      "rewards/rejected": -0.5514900088310242,
      "step": 247
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.9952482577785545e-05,
      "logits/chosen": -2.4535770416259766,
      "logits/rejected": -2.5108795166015625,
      "logps/chosen": -186.94288635253906,
      "logps/rejected": -191.04786682128906,
      "loss": 0.7335,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6674529314041138,
      "rewards/margins": -0.03079444169998169,
      "rewards/rejected": -0.6366585493087769,
      "step": 248
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.9951838488057134e-05,
      "logits/chosen": -2.2177467346191406,
      "logits/rejected": -2.3985226154327393,
      "logps/chosen": -151.24977111816406,
      "logps/rejected": -185.8676300048828,
      "loss": 0.6516,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3900182843208313,
      "rewards/margins": 0.13504652678966522,
      "rewards/rejected": -0.5250648260116577,
      "step": 249
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.9951190066616495e-05,
      "logits/chosen": -2.42427396774292,
      "logits/rejected": -2.4583323001861572,
      "logps/chosen": -255.53150939941406,
      "logps/rejected": -261.932373046875,
      "loss": 0.6442,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3976048231124878,
      "rewards/margins": 0.12972235679626465,
      "rewards/rejected": -0.5273271799087524,
      "step": 250
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.995053731357618e-05,
      "logits/chosen": -2.462376832962036,
      "logits/rejected": -2.568171262741089,
      "logps/chosen": -197.1337432861328,
      "logps/rejected": -202.1964569091797,
      "loss": 0.6419,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6016325354576111,
      "rewards/margins": 0.15742458403110504,
      "rewards/rejected": -0.7590572237968445,
      "step": 251
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.9949880229049526e-05,
      "logits/chosen": -2.4696521759033203,
      "logits/rejected": -2.4089298248291016,
      "logps/chosen": -187.97976684570312,
      "logps/rejected": -178.5655059814453,
      "loss": 0.7341,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.43453776836395264,
      "rewards/margins": -0.025840837508440018,
      "rewards/rejected": -0.4086969494819641,
      "step": 252
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.994921881315059e-05,
      "logits/chosen": -2.292558431625366,
      "logits/rejected": -2.199901580810547,
      "logps/chosen": -192.6279754638672,
      "logps/rejected": -203.34487915039062,
      "loss": 0.6863,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.48495134711265564,
      "rewards/margins": 0.07142490148544312,
      "rewards/rejected": -0.5563762784004211,
      "step": 253
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.9948553065994197e-05,
      "logits/chosen": -2.231633424758911,
      "logits/rejected": -2.2734503746032715,
      "logps/chosen": -184.4136962890625,
      "logps/rejected": -194.4268341064453,
      "loss": 0.6465,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.44051364064216614,
      "rewards/margins": 0.19739174842834473,
      "rewards/rejected": -0.6379053592681885,
      "step": 254
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.994788298769593e-05,
      "logits/chosen": -2.5150344371795654,
      "logits/rejected": -2.4598844051361084,
      "logps/chosen": -238.438232421875,
      "logps/rejected": -217.13897705078125,
      "loss": 0.6945,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.43908870220184326,
      "rewards/margins": 0.03722059726715088,
      "rewards/rejected": -0.47630926966667175,
      "step": 255
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.994720857837211e-05,
      "logits/chosen": -2.2310400009155273,
      "logits/rejected": -2.1882810592651367,
      "logps/chosen": -184.02981567382812,
      "logps/rejected": -208.0157470703125,
      "loss": 0.666,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5550325512886047,
      "rewards/margins": 0.08377734571695328,
      "rewards/rejected": -0.638809859752655,
      "step": 256
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.994652983813982e-05,
      "logits/chosen": -2.4377846717834473,
      "logits/rejected": -2.5074000358581543,
      "logps/chosen": -187.0503387451172,
      "logps/rejected": -195.54049682617188,
      "loss": 0.5873,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5530596971511841,
      "rewards/margins": 0.2584923803806305,
      "rewards/rejected": -0.811552107334137,
      "step": 257
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.994584676711689e-05,
      "logits/chosen": -2.464853286743164,
      "logits/rejected": -2.504178047180176,
      "logps/chosen": -186.5401611328125,
      "logps/rejected": -227.3590087890625,
      "loss": 0.7102,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5265690088272095,
      "rewards/margins": 0.009033482521772385,
      "rewards/rejected": -0.5356025695800781,
      "step": 258
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.994515936542191e-05,
      "logits/chosen": -2.2838244438171387,
      "logits/rejected": -2.3256828784942627,
      "logps/chosen": -173.54270935058594,
      "logps/rejected": -166.37547302246094,
      "loss": 0.7262,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4617432951927185,
      "rewards/margins": -0.03625304624438286,
      "rewards/rejected": -0.42549026012420654,
      "step": 259
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.99444676331742e-05,
      "logits/chosen": -2.407790422439575,
      "logits/rejected": -2.3871612548828125,
      "logps/chosen": -248.3179931640625,
      "logps/rejected": -235.79884338378906,
      "loss": 0.6761,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.458143413066864,
      "rewards/margins": 0.10666979104280472,
      "rewards/rejected": -0.5648132562637329,
      "step": 260
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.9943771570493856e-05,
      "logits/chosen": -2.560073137283325,
      "logits/rejected": -2.51522159576416,
      "logps/chosen": -191.05734252929688,
      "logps/rejected": -195.287841796875,
      "loss": 0.7288,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6246404051780701,
      "rewards/margins": -0.01798657327890396,
      "rewards/rejected": -0.6066538095474243,
      "step": 261
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.9943071177501724e-05,
      "logits/chosen": -2.364337682723999,
      "logits/rejected": -2.39058780670166,
      "logps/chosen": -200.97915649414062,
      "logps/rejected": -183.07151794433594,
      "loss": 0.6839,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5552209615707397,
      "rewards/margins": 0.09874990582466125,
      "rewards/rejected": -0.6539708375930786,
      "step": 262
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.994236645431938e-05,
      "logits/chosen": -2.6555843353271484,
      "logits/rejected": -2.7670748233795166,
      "logps/chosen": -244.5951690673828,
      "logps/rejected": -246.04124450683594,
      "loss": 0.688,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.45607852935791016,
      "rewards/margins": 0.08103760331869125,
      "rewards/rejected": -0.5371161103248596,
      "step": 263
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.994165740106918e-05,
      "logits/chosen": -2.527970314025879,
      "logits/rejected": -2.6416518688201904,
      "logps/chosen": -231.80966186523438,
      "logps/rejected": -246.3621826171875,
      "loss": 0.6086,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5444769263267517,
      "rewards/margins": 0.21878674626350403,
      "rewards/rejected": -0.7632635831832886,
      "step": 264
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.99409440178742e-05,
      "logits/chosen": -2.584487199783325,
      "logits/rejected": -2.5451977252960205,
      "logps/chosen": -184.02330017089844,
      "logps/rejected": -187.47161865234375,
      "loss": 0.7151,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6932749152183533,
      "rewards/margins": -0.005857221782207489,
      "rewards/rejected": -0.6874176263809204,
      "step": 265
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.9940226304858296e-05,
      "logits/chosen": -2.556795597076416,
      "logits/rejected": -2.5516929626464844,
      "logps/chosen": -187.25588989257812,
      "logps/rejected": -206.25067138671875,
      "loss": 0.7035,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.563089907169342,
      "rewards/margins": 0.009414192289113998,
      "rewards/rejected": -0.5725040435791016,
      "step": 266
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.993950426214606e-05,
      "logits/chosen": -2.3932271003723145,
      "logits/rejected": -2.393653154373169,
      "logps/chosen": -218.82334899902344,
      "logps/rejected": -226.3861541748047,
      "loss": 0.6841,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6887062788009644,
      "rewards/margins": 0.031964417546987534,
      "rewards/rejected": -0.720670759677887,
      "step": 267
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.993877788986285e-05,
      "logits/chosen": -2.3766613006591797,
      "logits/rejected": -2.507246971130371,
      "logps/chosen": -165.4411163330078,
      "logps/rejected": -199.56741333007812,
      "loss": 0.575,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.49406641721725464,
      "rewards/margins": 0.3596181869506836,
      "rewards/rejected": -0.853684663772583,
      "step": 268
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.9938047188134776e-05,
      "logits/chosen": -2.4775924682617188,
      "logits/rejected": -2.4857354164123535,
      "logps/chosen": -195.7847442626953,
      "logps/rejected": -216.16220092773438,
      "loss": 0.6795,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5620791912078857,
      "rewards/margins": 0.07040975242853165,
      "rewards/rejected": -0.632489025592804,
      "step": 269
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.993731215708866e-05,
      "logits/chosen": -2.637573719024658,
      "logits/rejected": -2.700226306915283,
      "logps/chosen": -200.697509765625,
      "logps/rejected": -214.4364471435547,
      "loss": 0.6027,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5127323269844055,
      "rewards/margins": 0.24725483357906342,
      "rewards/rejected": -0.7599871754646301,
      "step": 270
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.993657279685212e-05,
      "logits/chosen": -2.4737141132354736,
      "logits/rejected": -2.576711654663086,
      "logps/chosen": -198.31393432617188,
      "logps/rejected": -219.9713592529297,
      "loss": 0.7162,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7182321548461914,
      "rewards/margins": 0.04057084769010544,
      "rewards/rejected": -0.7588030099868774,
      "step": 271
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.9935829107553516e-05,
      "logits/chosen": -2.3163719177246094,
      "logits/rejected": -2.3124592304229736,
      "logps/chosen": -198.31076049804688,
      "logps/rejected": -210.8530731201172,
      "loss": 0.6581,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5568541288375854,
      "rewards/margins": 0.1273648738861084,
      "rewards/rejected": -0.6842190623283386,
      "step": 272
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.993508108932195e-05,
      "logits/chosen": -2.4512782096862793,
      "logits/rejected": -2.508169651031494,
      "logps/chosen": -202.17613220214844,
      "logps/rejected": -203.27789306640625,
      "loss": 0.6612,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.7900542616844177,
      "rewards/margins": 0.12749117612838745,
      "rewards/rejected": -0.91754549741745,
      "step": 273
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.9934328742287285e-05,
      "logits/chosen": -2.391414165496826,
      "logits/rejected": -2.3899242877960205,
      "logps/chosen": -231.99913024902344,
      "logps/rejected": -232.77243041992188,
      "loss": 0.7863,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6857910752296448,
      "rewards/margins": -0.05944066494703293,
      "rewards/rejected": -0.626350462436676,
      "step": 274
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.993357206658011e-05,
      "logits/chosen": -2.5768861770629883,
      "logits/rejected": -2.6738839149475098,
      "logps/chosen": -175.33010864257812,
      "logps/rejected": -185.26443481445312,
      "loss": 0.7157,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6001064777374268,
      "rewards/margins": 0.052334412932395935,
      "rewards/rejected": -0.6524409651756287,
      "step": 275
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.993281106233182e-05,
      "logits/chosen": -2.3842387199401855,
      "logits/rejected": -2.425611972808838,
      "logps/chosen": -165.09873962402344,
      "logps/rejected": -185.34361267089844,
      "loss": 0.5746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4550285339355469,
      "rewards/margins": 0.2833191156387329,
      "rewards/rejected": -0.738347589969635,
      "step": 276
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.9932045729674505e-05,
      "logits/chosen": -2.370333671569824,
      "logits/rejected": -2.343536376953125,
      "logps/chosen": -211.58517456054688,
      "logps/rejected": -191.9432373046875,
      "loss": 0.7727,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7500544786453247,
      "rewards/margins": -0.05222831293940544,
      "rewards/rejected": -0.697826087474823,
      "step": 277
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.993127606874104e-05,
      "logits/chosen": -2.3698928356170654,
      "logits/rejected": -2.459376573562622,
      "logps/chosen": -241.92343139648438,
      "logps/rejected": -265.6181945800781,
      "loss": 0.6437,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6877275705337524,
      "rewards/margins": 0.16285517811775208,
      "rewards/rejected": -0.8505828380584717,
      "step": 278
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.9930502079665025e-05,
      "logits/chosen": -2.3997507095336914,
      "logits/rejected": -2.4078052043914795,
      "logps/chosen": -228.6896514892578,
      "logps/rejected": -221.53338623046875,
      "loss": 0.7984,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6329948306083679,
      "rewards/margins": -0.1108212098479271,
      "rewards/rejected": -0.5221735835075378,
      "step": 279
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.9929723762580835e-05,
      "logits/chosen": -2.498244285583496,
      "logits/rejected": -2.571398973464966,
      "logps/chosen": -192.15635681152344,
      "logps/rejected": -176.19012451171875,
      "loss": 0.7497,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5479970574378967,
      "rewards/margins": -0.0442587286233902,
      "rewards/rejected": -0.5037383437156677,
      "step": 280
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.9928941117623604e-05,
      "logits/chosen": -2.6329376697540283,
      "logits/rejected": -2.635847806930542,
      "logps/chosen": -207.86293029785156,
      "logps/rejected": -224.18490600585938,
      "loss": 0.6649,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5313466787338257,
      "rewards/margins": 0.07021217793226242,
      "rewards/rejected": -0.6015588045120239,
      "step": 281
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.9928154144929175e-05,
      "logits/chosen": -2.5966103076934814,
      "logits/rejected": -2.571523666381836,
      "logps/chosen": -192.47384643554688,
      "logps/rejected": -180.02134704589844,
      "loss": 0.8799,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.8716793656349182,
      "rewards/margins": -0.2290811538696289,
      "rewards/rejected": -0.6425981521606445,
      "step": 282
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.9927362844634186e-05,
      "logits/chosen": -2.6513888835906982,
      "logits/rejected": -2.606253147125244,
      "logps/chosen": -208.56338500976562,
      "logps/rejected": -200.686279296875,
      "loss": 0.7609,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5643545389175415,
      "rewards/margins": -0.08666092902421951,
      "rewards/rejected": -0.4776935577392578,
      "step": 283
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.9926567216876e-05,
      "logits/chosen": -2.5372819900512695,
      "logits/rejected": -2.5824756622314453,
      "logps/chosen": -187.18386840820312,
      "logps/rejected": -211.16427612304688,
      "loss": 0.6838,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5520645380020142,
      "rewards/margins": 0.10136236250400543,
      "rewards/rejected": -0.6534268856048584,
      "step": 284
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.992576726179274e-05,
      "logits/chosen": -2.5787792205810547,
      "logits/rejected": -2.5688636302948,
      "logps/chosen": -199.59490966796875,
      "logps/rejected": -221.02023315429688,
      "loss": 0.7033,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5855346918106079,
      "rewards/margins": 0.010282933712005615,
      "rewards/rejected": -0.5958175659179688,
      "step": 285
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.9924962979523296e-05,
      "logits/chosen": -2.588792085647583,
      "logits/rejected": -2.5663416385650635,
      "logps/chosen": -197.04592895507812,
      "logps/rejected": -184.49290466308594,
      "loss": 0.6537,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.46529656648635864,
      "rewards/margins": 0.1208263412117958,
      "rewards/rejected": -0.5861228704452515,
      "step": 286
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.992415437020727e-05,
      "logits/chosen": -2.6553092002868652,
      "logits/rejected": -2.763751268386841,
      "logps/chosen": -217.5886993408203,
      "logps/rejected": -233.33489990234375,
      "loss": 0.6027,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4947764575481415,
      "rewards/margins": 0.2827147841453552,
      "rewards/rejected": -0.7774912118911743,
      "step": 287
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.992334143398506e-05,
      "logits/chosen": -2.5335421562194824,
      "logits/rejected": -2.600069999694824,
      "logps/chosen": -204.98912048339844,
      "logps/rejected": -212.0712890625,
      "loss": 0.6079,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5413306355476379,
      "rewards/margins": 0.22219431400299072,
      "rewards/rejected": -0.7635249495506287,
      "step": 288
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.992252417099778e-05,
      "logits/chosen": -2.638511896133423,
      "logits/rejected": -2.600456476211548,
      "logps/chosen": -172.59390258789062,
      "logps/rejected": -192.1689910888672,
      "loss": 0.7194,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5202851891517639,
      "rewards/margins": 0.017805464565753937,
      "rewards/rejected": -0.5380906462669373,
      "step": 289
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.992170258138732e-05,
      "logits/chosen": -2.5361521244049072,
      "logits/rejected": -2.4191856384277344,
      "logps/chosen": -177.23558044433594,
      "logps/rejected": -169.40321350097656,
      "loss": 0.6089,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20310896635055542,
      "rewards/margins": 0.2397383451461792,
      "rewards/rejected": -0.4428473711013794,
      "step": 290
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.99208766652963e-05,
      "logits/chosen": -2.727092742919922,
      "logits/rejected": -2.613562822341919,
      "logps/chosen": -228.9248504638672,
      "logps/rejected": -222.22886657714844,
      "loss": 0.6345,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5637404322624207,
      "rewards/margins": 0.21417354047298431,
      "rewards/rejected": -0.7779139280319214,
      "step": 291
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.99200464228681e-05,
      "logits/chosen": -2.530290126800537,
      "logits/rejected": -2.49137282371521,
      "logps/chosen": -218.17156982421875,
      "logps/rejected": -225.41952514648438,
      "loss": 0.7465,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.567755937576294,
      "rewards/margins": -0.07816261053085327,
      "rewards/rejected": -0.4895933270454407,
      "step": 292
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.9919211854246874e-05,
      "logits/chosen": -2.683086633682251,
      "logits/rejected": -2.588696002960205,
      "logps/chosen": -268.5683898925781,
      "logps/rejected": -255.58445739746094,
      "loss": 0.7069,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5820844769477844,
      "rewards/margins": 0.0005566291511058807,
      "rewards/rejected": -0.582641065120697,
      "step": 293
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.9918372959577486e-05,
      "logits/chosen": -2.7265396118164062,
      "logits/rejected": -2.7635843753814697,
      "logps/chosen": -206.09573364257812,
      "logps/rejected": -214.75784301757812,
      "loss": 0.6975,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6429769396781921,
      "rewards/margins": 0.1274566352367401,
      "rewards/rejected": -0.7704335451126099,
      "step": 294
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.9917529739005574e-05,
      "logits/chosen": -2.507643938064575,
      "logits/rejected": -2.4392926692962646,
      "logps/chosen": -219.877197265625,
      "logps/rejected": -217.41537475585938,
      "loss": 0.729,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5904738306999207,
      "rewards/margins": -0.04813346266746521,
      "rewards/rejected": -0.5423403382301331,
      "step": 295
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.991668219267752e-05,
      "logits/chosen": -2.613016128540039,
      "logits/rejected": -2.7239303588867188,
      "logps/chosen": -235.21534729003906,
      "logps/rejected": -266.6222839355469,
      "loss": 0.7173,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6593362092971802,
      "rewards/margins": 0.04644089192152023,
      "rewards/rejected": -0.7057771682739258,
      "step": 296
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.991583032074047e-05,
      "logits/chosen": -2.774909257888794,
      "logits/rejected": -2.7659873962402344,
      "logps/chosen": -250.04261779785156,
      "logps/rejected": -242.6505584716797,
      "loss": 0.7062,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6024019718170166,
      "rewards/margins": 0.010418819263577461,
      "rewards/rejected": -0.6128207445144653,
      "step": 297
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.99149741233423e-05,
      "logits/chosen": -2.447873592376709,
      "logits/rejected": -2.468458414077759,
      "logps/chosen": -214.55784606933594,
      "logps/rejected": -189.22238159179688,
      "loss": 0.6822,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6992676854133606,
      "rewards/margins": 0.09424494951963425,
      "rewards/rejected": -0.7935126423835754,
      "step": 298
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.9914113600631665e-05,
      "logits/chosen": -2.5641584396362305,
      "logits/rejected": -2.602008104324341,
      "logps/chosen": -310.73077392578125,
      "logps/rejected": -290.17352294921875,
      "loss": 0.6765,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5222407579421997,
      "rewards/margins": 0.06505537033081055,
      "rewards/rejected": -0.587296187877655,
      "step": 299
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.991324875275794e-05,
      "logits/chosen": -2.5343639850616455,
      "logits/rejected": -2.610719919204712,
      "logps/chosen": -199.84573364257812,
      "logps/rejected": -203.96478271484375,
      "loss": 0.718,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5667111873626709,
      "rewards/margins": -0.001658361405134201,
      "rewards/rejected": -0.5650528073310852,
      "step": 300
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.991237957987127e-05,
      "logits/chosen": -2.379605293273926,
      "logits/rejected": -2.3678669929504395,
      "logps/chosen": -211.86761474609375,
      "logps/rejected": -213.41464233398438,
      "loss": 0.6325,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.46346691250801086,
      "rewards/margins": 0.20269399881362915,
      "rewards/rejected": -0.6661609411239624,
      "step": 301
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.991150608212254e-05,
      "logits/chosen": -2.6828835010528564,
      "logits/rejected": -2.7126364707946777,
      "logps/chosen": -196.61216735839844,
      "logps/rejected": -223.1612548828125,
      "loss": 0.7503,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5514590740203857,
      "rewards/margins": -0.08642277121543884,
      "rewards/rejected": -0.4650362730026245,
      "step": 302
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.9910628259663404e-05,
      "logits/chosen": -2.4960732460021973,
      "logits/rejected": -2.608139991760254,
      "logps/chosen": -197.00277709960938,
      "logps/rejected": -196.33804321289062,
      "loss": 0.564,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35482868552207947,
      "rewards/margins": 0.38187745213508606,
      "rewards/rejected": -0.7367061376571655,
      "step": 303
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.990974611264625e-05,
      "logits/chosen": -2.3627309799194336,
      "logits/rejected": -2.3418447971343994,
      "logps/chosen": -232.52740478515625,
      "logps/rejected": -227.15753173828125,
      "loss": 0.6344,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6158957481384277,
      "rewards/margins": 0.18052013218402863,
      "rewards/rejected": -0.7964158654212952,
      "step": 304
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.990885964122421e-05,
      "logits/chosen": -2.594547748565674,
      "logits/rejected": -2.6074790954589844,
      "logps/chosen": -170.09300231933594,
      "logps/rejected": -190.0100860595703,
      "loss": 0.7467,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5137202739715576,
      "rewards/margins": -0.07406175136566162,
      "rewards/rejected": -0.439658522605896,
      "step": 305
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.990796884555119e-05,
      "logits/chosen": -2.632927417755127,
      "logits/rejected": -2.6694023609161377,
      "logps/chosen": -215.45489501953125,
      "logps/rejected": -203.8738555908203,
      "loss": 0.6641,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4411565661430359,
      "rewards/margins": 0.1269129067659378,
      "rewards/rejected": -0.5680694580078125,
      "step": 306
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.9907073725781836e-05,
      "logits/chosen": -2.4353699684143066,
      "logits/rejected": -2.537163734436035,
      "logps/chosen": -279.5519714355469,
      "logps/rejected": -290.08135986328125,
      "loss": 0.6756,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.32637274265289307,
      "rewards/margins": 0.053382109850645065,
      "rewards/rejected": -0.379754900932312,
      "step": 307
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.9906174282071535e-05,
      "logits/chosen": -2.586729049682617,
      "logits/rejected": -2.583314895629883,
      "logps/chosen": -197.04298400878906,
      "logps/rejected": -195.334228515625,
      "loss": 0.6718,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.515608549118042,
      "rewards/margins": 0.06315431743860245,
      "rewards/rejected": -0.5787628293037415,
      "step": 308
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.990527051457644e-05,
      "logits/chosen": -2.45725154876709,
      "logits/rejected": -2.445600748062134,
      "logps/chosen": -204.52867126464844,
      "logps/rejected": -290.3397216796875,
      "loss": 0.6908,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6991719007492065,
      "rewards/margins": 0.08316923677921295,
      "rewards/rejected": -0.7823411226272583,
      "step": 309
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.9904362423453446e-05,
      "logits/chosen": -2.604722738265991,
      "logits/rejected": -2.6120543479919434,
      "logps/chosen": -193.1224822998047,
      "logps/rejected": -189.89694213867188,
      "loss": 0.6076,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3250601589679718,
      "rewards/margins": 0.19382469356060028,
      "rewards/rejected": -0.5188848376274109,
      "step": 310
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.990345000886019e-05,
      "logits/chosen": -2.6838831901550293,
      "logits/rejected": -2.6718058586120605,
      "logps/chosen": -177.02090454101562,
      "logps/rejected": -173.667724609375,
      "loss": 0.6993,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5827906131744385,
      "rewards/margins": 0.027324199676513672,
      "rewards/rejected": -0.6101148128509521,
      "step": 311
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.990253327095509e-05,
      "logits/chosen": -2.60197377204895,
      "logits/rejected": -2.522291660308838,
      "logps/chosen": -173.03811645507812,
      "logps/rejected": -149.1508026123047,
      "loss": 0.7939,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4563007950782776,
      "rewards/margins": -0.13545696437358856,
      "rewards/rejected": -0.32084381580352783,
      "step": 312
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.9901612209897275e-05,
      "logits/chosen": -2.643686532974243,
      "logits/rejected": -2.6443119049072266,
      "logps/chosen": -187.93931579589844,
      "logps/rejected": -208.4308624267578,
      "loss": 0.6686,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6419711709022522,
      "rewards/margins": 0.10658866912126541,
      "rewards/rejected": -0.748559832572937,
      "step": 313
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.990068682584666e-05,
      "logits/chosen": -2.469681739807129,
      "logits/rejected": -2.4439382553100586,
      "logps/chosen": -186.7505340576172,
      "logps/rejected": -204.6378936767578,
      "loss": 0.7857,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5965012311935425,
      "rewards/margins": -0.12374071031808853,
      "rewards/rejected": -0.47276046872138977,
      "step": 314
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.989975711896388e-05,
      "logits/chosen": -2.3346047401428223,
      "logits/rejected": -2.323143720626831,
      "logps/chosen": -223.70867919921875,
      "logps/rejected": -238.58311462402344,
      "loss": 0.7061,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5982979536056519,
      "rewards/margins": -0.005420597270131111,
      "rewards/rejected": -0.5928773283958435,
      "step": 315
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.989882308941034e-05,
      "logits/chosen": -2.759549379348755,
      "logits/rejected": -2.7803878784179688,
      "logps/chosen": -239.48947143554688,
      "logps/rejected": -181.70718383789062,
      "loss": 0.6671,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.46423962712287903,
      "rewards/margins": 0.11677178740501404,
      "rewards/rejected": -0.5810113549232483,
      "step": 316
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.9897884737348196e-05,
      "logits/chosen": -2.485264778137207,
      "logits/rejected": -2.5983753204345703,
      "logps/chosen": -175.43971252441406,
      "logps/rejected": -219.08517456054688,
      "loss": 0.7124,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5487624406814575,
      "rewards/margins": 0.0002348199486732483,
      "rewards/rejected": -0.5489972829818726,
      "step": 317
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.989694206294035e-05,
      "logits/chosen": -2.6732518672943115,
      "logits/rejected": -2.6291115283966064,
      "logps/chosen": -193.59197998046875,
      "logps/rejected": -207.9779052734375,
      "loss": 0.6528,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4739716649055481,
      "rewards/margins": 0.12173515558242798,
      "rewards/rejected": -0.5957068204879761,
      "step": 318
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.989599506635044e-05,
      "logits/chosen": -2.4193854331970215,
      "logits/rejected": -2.565293073654175,
      "logps/chosen": -263.6156311035156,
      "logps/rejected": -251.44192504882812,
      "loss": 0.6307,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.47510460019111633,
      "rewards/margins": 0.25979098677635193,
      "rewards/rejected": -0.734895646572113,
      "step": 319
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.989504374774288e-05,
      "logits/chosen": -2.3681142330169678,
      "logits/rejected": -2.4691953659057617,
      "logps/chosen": -163.5777587890625,
      "logps/rejected": -194.8612060546875,
      "loss": 0.6185,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4559204876422882,
      "rewards/margins": 0.19214320182800293,
      "rewards/rejected": -0.648063600063324,
      "step": 320
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.989408810728281e-05,
      "logits/chosen": -2.477172613143921,
      "logits/rejected": -2.5207676887512207,
      "logps/chosen": -141.12867736816406,
      "logps/rejected": -146.79791259765625,
      "loss": 0.657,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.22462037205696106,
      "rewards/margins": 0.09705987572669983,
      "rewards/rejected": -0.3216802775859833,
      "step": 321
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.989312814513614e-05,
      "logits/chosen": -2.534242630004883,
      "logits/rejected": -2.47320294380188,
      "logps/chosen": -223.2980499267578,
      "logps/rejected": -224.9616241455078,
      "loss": 0.7372,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5472955107688904,
      "rewards/margins": -0.037168219685554504,
      "rewards/rejected": -0.5101273059844971,
      "step": 322
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.989216386146953e-05,
      "logits/chosen": -2.3438358306884766,
      "logits/rejected": -2.623857021331787,
      "logps/chosen": -190.59585571289062,
      "logps/rejected": -234.1730194091797,
      "loss": 0.6873,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3276049494743347,
      "rewards/margins": 0.06552110612392426,
      "rewards/rejected": -0.39312607049942017,
      "step": 323
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.9891195256450366e-05,
      "logits/chosen": -2.210477828979492,
      "logits/rejected": -2.195955991744995,
      "logps/chosen": -185.43902587890625,
      "logps/rejected": -224.94488525390625,
      "loss": 0.678,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38528165221214294,
      "rewards/margins": 0.09291733056306839,
      "rewards/rejected": -0.47819897532463074,
      "step": 324
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.989022233024681e-05,
      "logits/chosen": -2.7818360328674316,
      "logits/rejected": -2.764118194580078,
      "logps/chosen": -224.84347534179688,
      "logps/rejected": -205.856201171875,
      "loss": 0.6028,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3358863592147827,
      "rewards/margins": 0.23018960654735565,
      "rewards/rejected": -0.5660759806632996,
      "step": 325
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.9889245083027755e-05,
      "logits/chosen": -2.3800809383392334,
      "logits/rejected": -2.3793954849243164,
      "logps/chosen": -168.0996551513672,
      "logps/rejected": -197.4810791015625,
      "loss": 0.7001,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.36885035037994385,
      "rewards/margins": 0.015535619109869003,
      "rewards/rejected": -0.3843860328197479,
      "step": 326
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.988826351496287e-05,
      "logits/chosen": -2.5323662757873535,
      "logits/rejected": -2.5999341011047363,
      "logps/chosen": -215.6177978515625,
      "logps/rejected": -255.65231323242188,
      "loss": 0.6679,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6092446446418762,
      "rewards/margins": 0.09709976613521576,
      "rewards/rejected": -0.7063443660736084,
      "step": 327
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.988727762622255e-05,
      "logits/chosen": -2.7337067127227783,
      "logits/rejected": -2.742793083190918,
      "logps/chosen": -214.25506591796875,
      "logps/rejected": -221.33468627929688,
      "loss": 0.6614,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6302060484886169,
      "rewards/margins": 0.12371520698070526,
      "rewards/rejected": -0.7539212107658386,
      "step": 328
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.9886287416977936e-05,
      "logits/chosen": -2.510671377182007,
      "logits/rejected": -2.5760793685913086,
      "logps/chosen": -169.99656677246094,
      "logps/rejected": -181.1496124267578,
      "loss": 0.6957,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4545098841190338,
      "rewards/margins": 0.014530147425830364,
      "rewards/rejected": -0.46904003620147705,
      "step": 329
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.988529288740096e-05,
      "logits/chosen": -2.6438777446746826,
      "logits/rejected": -2.6174240112304688,
      "logps/chosen": -187.69232177734375,
      "logps/rejected": -198.16409301757812,
      "loss": 0.6247,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.37409377098083496,
      "rewards/margins": 0.1910334676504135,
      "rewards/rejected": -0.5651272535324097,
      "step": 330
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.9884294037664245e-05,
      "logits/chosen": -2.456332206726074,
      "logits/rejected": -2.4327914714813232,
      "logps/chosen": -221.3645782470703,
      "logps/rejected": -220.4224853515625,
      "loss": 0.726,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9011685848236084,
      "rewards/margins": 0.07650195062160492,
      "rewards/rejected": -0.9776705503463745,
      "step": 331
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.988329086794122e-05,
      "logits/chosen": -2.463491439819336,
      "logits/rejected": -2.49444842338562,
      "logps/chosen": -197.96144104003906,
      "logps/rejected": -214.404296875,
      "loss": 0.6547,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.42488348484039307,
      "rewards/margins": 0.11507010459899902,
      "rewards/rejected": -0.5399536490440369,
      "step": 332
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.9882283378406015e-05,
      "logits/chosen": -2.5106682777404785,
      "logits/rejected": -2.5157594680786133,
      "logps/chosen": -167.09947204589844,
      "logps/rejected": -174.6180419921875,
      "loss": 0.6179,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4473687410354614,
      "rewards/margins": 0.22422267496585846,
      "rewards/rejected": -0.6715914011001587,
      "step": 333
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.988127156923355e-05,
      "logits/chosen": -2.4497499465942383,
      "logits/rejected": -2.491909980773926,
      "logps/chosen": -167.1006622314453,
      "logps/rejected": -165.177978515625,
      "loss": 0.6855,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5522390604019165,
      "rewards/margins": 0.06803999096155167,
      "rewards/rejected": -0.6202789545059204,
      "step": 334
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.9880255440599476e-05,
      "logits/chosen": -2.445422649383545,
      "logits/rejected": -2.4025139808654785,
      "logps/chosen": -199.83444213867188,
      "logps/rejected": -203.16355895996094,
      "loss": 0.7404,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.658203125,
      "rewards/margins": -0.06064491346478462,
      "rewards/rejected": -0.5975580811500549,
      "step": 335
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.987923499268018e-05,
      "logits/chosen": -2.540513753890991,
      "logits/rejected": -2.6426820755004883,
      "logps/chosen": -181.93406677246094,
      "logps/rejected": -203.6474609375,
      "loss": 0.6465,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45222824811935425,
      "rewards/margins": 0.17588049173355103,
      "rewards/rejected": -0.6281087398529053,
      "step": 336
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.987821022565284e-05,
      "logits/chosen": -2.608978509902954,
      "logits/rejected": -2.632282257080078,
      "logps/chosen": -195.6852569580078,
      "logps/rejected": -216.0919952392578,
      "loss": 0.6137,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4888995289802551,
      "rewards/margins": 0.3496631681919098,
      "rewards/rejected": -0.8385627269744873,
      "step": 337
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.987718113969534e-05,
      "logits/chosen": -2.637908458709717,
      "logits/rejected": -2.6849193572998047,
      "logps/chosen": -307.9834289550781,
      "logps/rejected": -321.95867919921875,
      "loss": 0.5909,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4099658131599426,
      "rewards/margins": 0.3652358055114746,
      "rewards/rejected": -0.7752016186714172,
      "step": 338
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.9876147734986335e-05,
      "logits/chosen": -2.6805038452148438,
      "logits/rejected": -2.659087657928467,
      "logps/chosen": -235.20855712890625,
      "logps/rejected": -222.38107299804688,
      "loss": 0.7633,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7779167294502258,
      "rewards/margins": -0.11177889257669449,
      "rewards/rejected": -0.6661379337310791,
      "step": 339
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.987511001170523e-05,
      "logits/chosen": -2.4979588985443115,
      "logits/rejected": -2.509103775024414,
      "logps/chosen": -205.0665283203125,
      "logps/rejected": -197.5661163330078,
      "loss": 0.584,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.603629469871521,
      "rewards/margins": 0.3201472759246826,
      "rewards/rejected": -0.9237766861915588,
      "step": 340
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.987406797003218e-05,
      "logits/chosen": -2.6034300327301025,
      "logits/rejected": -2.606010913848877,
      "logps/chosen": -212.59959411621094,
      "logps/rejected": -266.6811218261719,
      "loss": 0.7099,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.7883586287498474,
      "rewards/margins": 0.047954261302948,
      "rewards/rejected": -0.8363128900527954,
      "step": 341
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.987302161014808e-05,
      "logits/chosen": -2.26737380027771,
      "logits/rejected": -2.2702267169952393,
      "logps/chosen": -149.69259643554688,
      "logps/rejected": -207.59146118164062,
      "loss": 0.718,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6731799244880676,
      "rewards/margins": 0.02036801353096962,
      "rewards/rejected": -0.6935478448867798,
      "step": 342
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.9871970932234586e-05,
      "logits/chosen": -2.3482635021209717,
      "logits/rejected": -2.363096237182617,
      "logps/chosen": -196.5614013671875,
      "logps/rejected": -206.21414184570312,
      "loss": 0.6163,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5754539966583252,
      "rewards/margins": 0.2127307951450348,
      "rewards/rejected": -0.7881847620010376,
      "step": 343
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.9870915936474095e-05,
      "logits/chosen": -2.7459514141082764,
      "logits/rejected": -2.7396438121795654,
      "logps/chosen": -206.6090087890625,
      "logps/rejected": -211.6893768310547,
      "loss": 0.6763,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.7424849271774292,
      "rewards/margins": 0.08226263523101807,
      "rewards/rejected": -0.8247475624084473,
      "step": 344
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.986985662304976e-05,
      "logits/chosen": -2.769526243209839,
      "logits/rejected": -2.807554244995117,
      "logps/chosen": -184.0075225830078,
      "logps/rejected": -225.2920379638672,
      "loss": 0.6217,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7790610194206238,
      "rewards/margins": 0.24687500298023224,
      "rewards/rejected": -1.0259360074996948,
      "step": 345
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.9868792992145484e-05,
      "logits/chosen": -2.6607320308685303,
      "logits/rejected": -2.6989316940307617,
      "logps/chosen": -218.40530395507812,
      "logps/rejected": -227.24240112304688,
      "loss": 0.7374,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6406981945037842,
      "rewards/margins": 0.009773064404726028,
      "rewards/rejected": -0.6504712104797363,
      "step": 346
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.9867725043945904e-05,
      "logits/chosen": -2.5469489097595215,
      "logits/rejected": -2.539520740509033,
      "logps/chosen": -222.92864990234375,
      "logps/rejected": -236.92977905273438,
      "loss": 0.6992,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.640974223613739,
      "rewards/margins": 0.09771876782178879,
      "rewards/rejected": -0.7386929988861084,
      "step": 347
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.9866652778636436e-05,
      "logits/chosen": -2.545572280883789,
      "logits/rejected": -2.541029453277588,
      "logps/chosen": -182.341552734375,
      "logps/rejected": -197.535888671875,
      "loss": 0.7149,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.7396100163459778,
      "rewards/margins": 0.06020259112119675,
      "rewards/rejected": -0.7998126149177551,
      "step": 348
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.986557619640322e-05,
      "logits/chosen": -2.479189157485962,
      "logits/rejected": -2.504956007003784,
      "logps/chosen": -184.10472106933594,
      "logps/rejected": -184.99070739746094,
      "loss": 0.5709,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6914107799530029,
      "rewards/margins": 0.33003267645835876,
      "rewards/rejected": -1.021443486213684,
      "step": 349
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.986449529743314e-05,
      "logits/chosen": -2.7267708778381348,
      "logits/rejected": -2.711848735809326,
      "logps/chosen": -225.98228454589844,
      "logps/rejected": -204.31724548339844,
      "loss": 0.7177,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.8359745740890503,
      "rewards/margins": 0.011185385286808014,
      "rewards/rejected": -0.8471599221229553,
      "step": 350
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.9863410081913875e-05,
      "logits/chosen": -2.691619396209717,
      "logits/rejected": -2.669311285018921,
      "logps/chosen": -157.73721313476562,
      "logps/rejected": -161.97061157226562,
      "loss": 0.7285,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5839475393295288,
      "rewards/margins": -0.027979355305433273,
      "rewards/rejected": -0.555968165397644,
      "step": 351
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.98623205500338e-05,
      "logits/chosen": -2.6664159297943115,
      "logits/rejected": -2.653743267059326,
      "logps/chosen": -155.03836059570312,
      "logps/rejected": -141.4946746826172,
      "loss": 0.7569,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5298622250556946,
      "rewards/margins": -0.08286149799823761,
      "rewards/rejected": -0.44700077176094055,
      "step": 352
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.986122670198205e-05,
      "logits/chosen": -2.588876485824585,
      "logits/rejected": -2.706108808517456,
      "logps/chosen": -170.23751831054688,
      "logps/rejected": -172.2996826171875,
      "loss": 0.8284,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9628427624702454,
      "rewards/margins": -0.18341386318206787,
      "rewards/rejected": -0.779429018497467,
      "step": 353
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.9860128537948555e-05,
      "logits/chosen": -2.8683338165283203,
      "logits/rejected": -2.814868211746216,
      "logps/chosen": -194.464599609375,
      "logps/rejected": -197.70077514648438,
      "loss": 0.7405,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6296394467353821,
      "rewards/margins": -0.0025831498205661774,
      "rewards/rejected": -0.6270563006401062,
      "step": 354
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.9859026058123925e-05,
      "logits/chosen": -2.978065013885498,
      "logits/rejected": -2.9368367195129395,
      "logps/chosen": -217.73974609375,
      "logps/rejected": -220.19183349609375,
      "loss": 0.7014,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6989356279373169,
      "rewards/margins": 0.007979679852724075,
      "rewards/rejected": -0.7069153189659119,
      "step": 355
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.985791926269958e-05,
      "logits/chosen": -2.666344165802002,
      "logits/rejected": -2.8283472061157227,
      "logps/chosen": -163.43722534179688,
      "logps/rejected": -190.4180908203125,
      "loss": 0.6778,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5998314023017883,
      "rewards/margins": 0.11102181673049927,
      "rewards/rejected": -0.7108532190322876,
      "step": 356
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.985680815186764e-05,
      "logits/chosen": -2.6322436332702637,
      "logits/rejected": -2.6792352199554443,
      "logps/chosen": -225.80972290039062,
      "logps/rejected": -221.8551025390625,
      "loss": 0.6464,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6963925957679749,
      "rewards/margins": 0.1484571099281311,
      "rewards/rejected": -0.844849705696106,
      "step": 357
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.985569272582101e-05,
      "logits/chosen": -2.829981803894043,
      "logits/rejected": -2.8077688217163086,
      "logps/chosen": -245.94029235839844,
      "logps/rejected": -279.1697082519531,
      "loss": 0.6446,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6325417757034302,
      "rewards/margins": 0.1735651046037674,
      "rewards/rejected": -0.8061069250106812,
      "step": 358
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.9854572984753334e-05,
      "logits/chosen": -2.8024842739105225,
      "logits/rejected": -2.750588893890381,
      "logps/chosen": -161.65240478515625,
      "logps/rejected": -201.25054931640625,
      "loss": 0.6502,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.46800869703292847,
      "rewards/margins": 0.12990553677082062,
      "rewards/rejected": -0.5979142189025879,
      "step": 359
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.985344892885899e-05,
      "logits/chosen": -2.347231149673462,
      "logits/rejected": -2.4064972400665283,
      "logps/chosen": -158.72801208496094,
      "logps/rejected": -167.15805053710938,
      "loss": 0.7264,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7969176173210144,
      "rewards/margins": 0.016100093722343445,
      "rewards/rejected": -0.8130176663398743,
      "step": 360
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.985232055833313e-05,
      "logits/chosen": -2.603712320327759,
      "logits/rejected": -2.6721928119659424,
      "logps/chosen": -211.3256378173828,
      "logps/rejected": -189.5877685546875,
      "loss": 0.7658,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6097235679626465,
      "rewards/margins": -0.10059280693531036,
      "rewards/rejected": -0.5091307163238525,
      "step": 361
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.985118787337164e-05,
      "logits/chosen": -2.6673433780670166,
      "logits/rejected": -2.766993284225464,
      "logps/chosen": -162.41415405273438,
      "logps/rejected": -218.15077209472656,
      "loss": 0.6236,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5241696238517761,
      "rewards/margins": 0.28995370864868164,
      "rewards/rejected": -0.8141233921051025,
      "step": 362
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.985005087417115e-05,
      "logits/chosen": -2.4291069507598877,
      "logits/rejected": -2.382821798324585,
      "logps/chosen": -221.48736572265625,
      "logps/rejected": -205.37940979003906,
      "loss": 0.6503,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4502621591091156,
      "rewards/margins": 0.2195020467042923,
      "rewards/rejected": -0.6697642207145691,
      "step": 363
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.984890956092905e-05,
      "logits/chosen": -2.5559186935424805,
      "logits/rejected": -2.496387481689453,
      "logps/chosen": -253.63047790527344,
      "logps/rejected": -244.56698608398438,
      "loss": 0.9484,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.9435632228851318,
      "rewards/margins": -0.268410325050354,
      "rewards/rejected": -0.6751528978347778,
      "step": 364
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.984776393384348e-05,
      "logits/chosen": -2.407367706298828,
      "logits/rejected": -2.3134310245513916,
      "logps/chosen": -173.61795043945312,
      "logps/rejected": -157.41207885742188,
      "loss": 0.7949,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5948637127876282,
      "rewards/margins": -0.14962351322174072,
      "rewards/rejected": -0.44524019956588745,
      "step": 365
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.984661399311332e-05,
      "logits/chosen": -2.6404154300689697,
      "logits/rejected": -2.7036595344543457,
      "logps/chosen": -155.3403778076172,
      "logps/rejected": -141.01382446289062,
      "loss": 0.7475,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4795231521129608,
      "rewards/margins": -0.06228278949856758,
      "rewards/rejected": -0.4172403812408447,
      "step": 366
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.9845459738938204e-05,
      "logits/chosen": -2.5403473377227783,
      "logits/rejected": -2.5674545764923096,
      "logps/chosen": -154.85910034179688,
      "logps/rejected": -143.22906494140625,
      "loss": 0.6505,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4790140390396118,
      "rewards/margins": 0.1586410105228424,
      "rewards/rejected": -0.6376550197601318,
      "step": 367
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.9844301171518516e-05,
      "logits/chosen": -2.6081302165985107,
      "logits/rejected": -2.641979217529297,
      "logps/chosen": -185.13006591796875,
      "logps/rejected": -198.2803955078125,
      "loss": 0.6726,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3776107430458069,
      "rewards/margins": 0.08190791308879852,
      "rewards/rejected": -0.4595187306404114,
      "step": 368
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.984313829105538e-05,
      "logits/chosen": -2.6662757396698,
      "logits/rejected": -2.7398993968963623,
      "logps/chosen": -201.8738555908203,
      "logps/rejected": -195.76390075683594,
      "loss": 0.5956,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38797736167907715,
      "rewards/margins": 0.2769092321395874,
      "rewards/rejected": -0.6648865342140198,
      "step": 369
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.984197109775068e-05,
      "logits/chosen": -2.4933226108551025,
      "logits/rejected": -2.484591245651245,
      "logps/chosen": -202.7592315673828,
      "logps/rejected": -199.02105712890625,
      "loss": 0.6536,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.35886555910110474,
      "rewards/margins": 0.1531214416027069,
      "rewards/rejected": -0.511987030506134,
      "step": 370
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.984079959180705e-05,
      "logits/chosen": -2.3560287952423096,
      "logits/rejected": -2.385936975479126,
      "logps/chosen": -198.06044006347656,
      "logps/rejected": -203.5518798828125,
      "loss": 0.7203,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.35552191734313965,
      "rewards/margins": 0.03736201673746109,
      "rewards/rejected": -0.39288395643234253,
      "step": 371
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.983962377342786e-05,
      "logits/chosen": -2.456765651702881,
      "logits/rejected": -2.522660970687866,
      "logps/chosen": -236.66744995117188,
      "logps/rejected": -241.58963012695312,
      "loss": 0.6049,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.551971435546875,
      "rewards/margins": 0.30021384358406067,
      "rewards/rejected": -0.8521853089332581,
      "step": 372
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.983844364281723e-05,
      "logits/chosen": -2.5745279788970947,
      "logits/rejected": -2.5961902141571045,
      "logps/chosen": -186.73411560058594,
      "logps/rejected": -194.74974060058594,
      "loss": 0.6354,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.312610000371933,
      "rewards/margins": 0.14659735560417175,
      "rewards/rejected": -0.45920735597610474,
      "step": 373
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.983725920018004e-05,
      "logits/chosen": -2.3747611045837402,
      "logits/rejected": -2.4170022010803223,
      "logps/chosen": -179.91162109375,
      "logps/rejected": -170.89920043945312,
      "loss": 0.6529,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.43839430809020996,
      "rewards/margins": 0.16247621178627014,
      "rewards/rejected": -0.6008704900741577,
      "step": 374
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.9836070445721924e-05,
      "logits/chosen": -2.573714017868042,
      "logits/rejected": -2.567589282989502,
      "logps/chosen": -204.4001922607422,
      "logps/rejected": -197.39869689941406,
      "loss": 0.751,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.350676029920578,
      "rewards/margins": -0.06606191396713257,
      "rewards/rejected": -0.28461411595344543,
      "step": 375
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.983487737964924e-05,
      "logits/chosen": -2.398193597793579,
      "logits/rejected": -2.4593253135681152,
      "logps/chosen": -199.0366973876953,
      "logps/rejected": -199.51275634765625,
      "loss": 0.6772,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.397927463054657,
      "rewards/margins": 0.08031069487333298,
      "rewards/rejected": -0.47823822498321533,
      "step": 376
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.9833680002169105e-05,
      "logits/chosen": -2.5120034217834473,
      "logits/rejected": -2.553926706314087,
      "logps/chosen": -226.15040588378906,
      "logps/rejected": -249.55255126953125,
      "loss": 0.6996,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3695937395095825,
      "rewards/margins": 0.0394027940928936,
      "rewards/rejected": -0.4089965522289276,
      "step": 377
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.983247831348939e-05,
      "logits/chosen": -2.289033889770508,
      "logits/rejected": -2.2501816749572754,
      "logps/chosen": -201.75392150878906,
      "logps/rejected": -217.8618621826172,
      "loss": 0.6645,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28367459774017334,
      "rewards/margins": 0.1500779092311859,
      "rewards/rejected": -0.43375247716903687,
      "step": 378
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.9831272313818716e-05,
      "logits/chosen": -2.4449117183685303,
      "logits/rejected": -2.522752046585083,
      "logps/chosen": -201.63587951660156,
      "logps/rejected": -215.33270263671875,
      "loss": 0.6512,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23286518454551697,
      "rewards/margins": 0.127617746591568,
      "rewards/rejected": -0.36048293113708496,
      "step": 379
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.983006200336645e-05,
      "logits/chosen": -2.4317235946655273,
      "logits/rejected": -2.3696205615997314,
      "logps/chosen": -243.03509521484375,
      "logps/rejected": -186.61427307128906,
      "loss": 0.7762,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5067302584648132,
      "rewards/margins": -0.11989758908748627,
      "rewards/rejected": -0.3868326246738434,
      "step": 380
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.98288473823427e-05,
      "logits/chosen": -2.4131155014038086,
      "logits/rejected": -2.498098611831665,
      "logps/chosen": -190.73397827148438,
      "logps/rejected": -213.9364471435547,
      "loss": 0.6336,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24977169930934906,
      "rewards/margins": 0.1747249811887741,
      "rewards/rejected": -0.42449668049812317,
      "step": 381
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.982762845095833e-05,
      "logits/chosen": -2.5675809383392334,
      "logits/rejected": -2.6289899349212646,
      "logps/chosen": -233.9481658935547,
      "logps/rejected": -245.3311767578125,
      "loss": 0.6879,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3384949564933777,
      "rewards/margins": 0.07099074125289917,
      "rewards/rejected": -0.40948566794395447,
      "step": 382
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.982640520942494e-05,
      "logits/chosen": -2.3582189083099365,
      "logits/rejected": -2.405932903289795,
      "logps/chosen": -157.98828125,
      "logps/rejected": -179.5477294921875,
      "loss": 0.6256,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.30075955390930176,
      "rewards/margins": 0.2314756065607071,
      "rewards/rejected": -0.5322352051734924,
      "step": 383
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.9825177657954914e-05,
      "logits/chosen": -2.6346352100372314,
      "logits/rejected": -2.7042407989501953,
      "logps/chosen": -174.1151123046875,
      "logps/rejected": -203.845458984375,
      "loss": 0.6727,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3591979742050171,
      "rewards/margins": 0.10004588216543198,
      "rewards/rejected": -0.45924389362335205,
      "step": 384
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.982394579676133e-05,
      "logits/chosen": -2.3420321941375732,
      "logits/rejected": -2.3010876178741455,
      "logps/chosen": -186.43467712402344,
      "logps/rejected": -183.117431640625,
      "loss": 0.6836,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3466501235961914,
      "rewards/margins": 0.06071464717388153,
      "rewards/rejected": -0.40736478567123413,
      "step": 385
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.982270962605806e-05,
      "logits/chosen": -2.4120569229125977,
      "logits/rejected": -2.449502944946289,
      "logps/chosen": -168.2112579345703,
      "logps/rejected": -227.885986328125,
      "loss": 0.5666,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22820252180099487,
      "rewards/margins": 0.35979852080345154,
      "rewards/rejected": -0.588001012802124,
      "step": 386
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.9821469146059704e-05,
      "logits/chosen": -2.3874173164367676,
      "logits/rejected": -2.4257519245147705,
      "logps/chosen": -158.00424194335938,
      "logps/rejected": -176.4921417236328,
      "loss": 0.6581,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28017669916152954,
      "rewards/margins": 0.168703094124794,
      "rewards/rejected": -0.44887974858283997,
      "step": 387
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.982022435698161e-05,
      "logits/chosen": -2.444685697555542,
      "logits/rejected": -2.653350591659546,
      "logps/chosen": -207.88034057617188,
      "logps/rejected": -231.86582946777344,
      "loss": 0.7048,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5877898931503296,
      "rewards/margins": 0.10215066373348236,
      "rewards/rejected": -0.6899405121803284,
      "step": 388
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.981897525903988e-05,
      "logits/chosen": -2.5300025939941406,
      "logits/rejected": -2.4865880012512207,
      "logps/chosen": -155.8424835205078,
      "logps/rejected": -152.6171875,
      "loss": 0.8172,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.5343464612960815,
      "rewards/margins": -0.17394664883613586,
      "rewards/rejected": -0.3603998124599457,
      "step": 389
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.981772185245135e-05,
      "logits/chosen": -2.61796236038208,
      "logits/rejected": -2.662163734436035,
      "logps/chosen": -198.32403564453125,
      "logps/rejected": -198.2169647216797,
      "loss": 0.751,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4620332419872284,
      "rewards/margins": -0.09868823736906052,
      "rewards/rejected": -0.3633449673652649,
      "step": 390
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.981646413743363e-05,
      "logits/chosen": -2.386425733566284,
      "logits/rejected": -2.452526092529297,
      "logps/chosen": -187.49838256835938,
      "logps/rejected": -181.8330535888672,
      "loss": 0.606,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27474457025527954,
      "rewards/margins": 0.2754724621772766,
      "rewards/rejected": -0.5502170324325562,
      "step": 391
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.981520211420506e-05,
      "logits/chosen": -2.4169254302978516,
      "logits/rejected": -2.4883086681365967,
      "logps/chosen": -175.3878173828125,
      "logps/rejected": -210.00119018554688,
      "loss": 0.7642,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.532840371131897,
      "rewards/margins": -0.09793217480182648,
      "rewards/rejected": -0.4349081814289093,
      "step": 392
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.9813935782984724e-05,
      "logits/chosen": -2.528134346008301,
      "logits/rejected": -2.491497039794922,
      "logps/chosen": -239.28878784179688,
      "logps/rejected": -241.08883666992188,
      "loss": 0.7798,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6152268648147583,
      "rewards/margins": -0.10364706814289093,
      "rewards/rejected": -0.5115798711776733,
      "step": 393
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.9812665143992466e-05,
      "logits/chosen": -2.5107626914978027,
      "logits/rejected": -2.591826915740967,
      "logps/chosen": -146.5967559814453,
      "logps/rejected": -154.36585998535156,
      "loss": 0.6751,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3082922697067261,
      "rewards/margins": 0.11212008446455002,
      "rewards/rejected": -0.4204123616218567,
      "step": 394
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.981139019744887e-05,
      "logits/chosen": -2.5607235431671143,
      "logits/rejected": -2.562152862548828,
      "logps/chosen": -185.95343017578125,
      "logps/rejected": -194.59483337402344,
      "loss": 0.6469,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2080746442079544,
      "rewards/margins": 0.12636929750442505,
      "rewards/rejected": -0.33444392681121826,
      "step": 395
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.981011094357527e-05,
      "logits/chosen": -2.4996774196624756,
      "logits/rejected": -2.5331335067749023,
      "logps/chosen": -215.7071075439453,
      "logps/rejected": -231.0633087158203,
      "loss": 0.7228,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4276806116104126,
      "rewards/margins": 0.03171852231025696,
      "rewards/rejected": -0.45939913392066956,
      "step": 396
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.980882738259376e-05,
      "logits/chosen": -2.3114399909973145,
      "logits/rejected": -2.3782169818878174,
      "logps/chosen": -169.04246520996094,
      "logps/rejected": -188.5185089111328,
      "loss": 0.7661,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.31136631965637207,
      "rewards/margins": -0.09910126030445099,
      "rewards/rejected": -0.21226505935192108,
      "step": 397
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.980753951472715e-05,
      "logits/chosen": -2.466881275177002,
      "logits/rejected": -2.5022642612457275,
      "logps/chosen": -168.63385009765625,
      "logps/rejected": -192.87521362304688,
      "loss": 0.6487,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3558157682418823,
      "rewards/margins": 0.13948221504688263,
      "rewards/rejected": -0.49529796838760376,
      "step": 398
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.980624734019903e-05,
      "logits/chosen": -2.5671474933624268,
      "logits/rejected": -2.6410720348358154,
      "logps/chosen": -183.6251220703125,
      "logps/rejected": -214.81272888183594,
      "loss": 0.7078,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4311671555042267,
      "rewards/margins": 0.0398234948515892,
      "rewards/rejected": -0.4709906280040741,
      "step": 399
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.980495085923372e-05,
      "logits/chosen": -2.602818012237549,
      "logits/rejected": -2.5820987224578857,
      "logps/chosen": -167.52012634277344,
      "logps/rejected": -180.94418334960938,
      "loss": 0.7179,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.41106754541397095,
      "rewards/margins": 0.013390736654400826,
      "rewards/rejected": -0.42445826530456543,
      "step": 400
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.980365007205631e-05,
      "logits/chosen": -2.4487011432647705,
      "logits/rejected": -2.5343518257141113,
      "logps/chosen": -201.9692840576172,
      "logps/rejected": -244.95530700683594,
      "loss": 0.5837,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3986441195011139,
      "rewards/margins": 0.3018190264701843,
      "rewards/rejected": -0.700463056564331,
      "step": 401
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.980234497889259e-05,
      "logits/chosen": -1.9015774726867676,
      "logits/rejected": -1.9911203384399414,
      "logps/chosen": -165.83837890625,
      "logps/rejected": -178.608154296875,
      "loss": 0.7531,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.47146371006965637,
      "rewards/margins": -0.03502827137708664,
      "rewards/rejected": -0.43643543124198914,
      "step": 402
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.980103557996915e-05,
      "logits/chosen": -2.609720468521118,
      "logits/rejected": -2.631443738937378,
      "logps/chosen": -235.0199737548828,
      "logps/rejected": -234.86245727539062,
      "loss": 0.6376,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2306605726480484,
      "rewards/margins": 0.16340221464633942,
      "rewards/rejected": -0.3940627872943878,
      "step": 403
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.9799721875513306e-05,
      "logits/chosen": -2.283406972885132,
      "logits/rejected": -2.386524200439453,
      "logps/chosen": -226.02059936523438,
      "logps/rejected": -220.51963806152344,
      "loss": 0.7208,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5439196228981018,
      "rewards/margins": 0.06111856549978256,
      "rewards/rejected": -0.6050382256507874,
      "step": 404
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.979840386575311e-05,
      "logits/chosen": -2.2238452434539795,
      "logits/rejected": -2.3374826908111572,
      "logps/chosen": -169.82667541503906,
      "logps/rejected": -197.92726135253906,
      "loss": 0.7292,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.46263885498046875,
      "rewards/margins": 0.007239609956741333,
      "rewards/rejected": -0.46987849473953247,
      "step": 405
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.979708155091737e-05,
      "logits/chosen": -2.381389856338501,
      "logits/rejected": -2.358883857727051,
      "logps/chosen": -153.38771057128906,
      "logps/rejected": -147.0846405029297,
      "loss": 0.6555,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2459157109260559,
      "rewards/margins": 0.1274775117635727,
      "rewards/rejected": -0.3733932375907898,
      "step": 406
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.979575493123566e-05,
      "logits/chosen": -2.4942612648010254,
      "logits/rejected": -2.4526309967041016,
      "logps/chosen": -222.35780334472656,
      "logps/rejected": -205.19630432128906,
      "loss": 0.7278,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4296140670776367,
      "rewards/margins": -0.018014922738075256,
      "rewards/rejected": -0.41159915924072266,
      "step": 407
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.979442400693827e-05,
      "logits/chosen": -2.4821577072143555,
      "logits/rejected": -2.4502267837524414,
      "logps/chosen": -157.32427978515625,
      "logps/rejected": -167.9297332763672,
      "loss": 0.7119,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4199235439300537,
      "rewards/margins": 0.024679476395249367,
      "rewards/rejected": -0.4446030259132385,
      "step": 408
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.979308877825626e-05,
      "logits/chosen": -2.1102793216705322,
      "logits/rejected": -2.100888729095459,
      "logps/chosen": -235.02105712890625,
      "logps/rejected": -232.0230712890625,
      "loss": 0.747,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4274646043777466,
      "rewards/margins": 0.0070232609286904335,
      "rewards/rejected": -0.43448784947395325,
      "step": 409
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.9791749245421434e-05,
      "logits/chosen": -2.4029147624969482,
      "logits/rejected": -2.436694622039795,
      "logps/chosen": -160.94296264648438,
      "logps/rejected": -219.91183471679688,
      "loss": 0.7659,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6857771873474121,
      "rewards/margins": -0.07571852207183838,
      "rewards/rejected": -0.6100587248802185,
      "step": 410
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.979040540866632e-05,
      "logits/chosen": -2.423585891723633,
      "logits/rejected": -2.583218574523926,
      "logps/chosen": -152.8751220703125,
      "logps/rejected": -195.98130798339844,
      "loss": 0.6592,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.29784753918647766,
      "rewards/margins": 0.13703800737857819,
      "rewards/rejected": -0.43488556146621704,
      "step": 411
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.9789057268224234e-05,
      "logits/chosen": -2.0688211917877197,
      "logits/rejected": -2.1250336170196533,
      "logps/chosen": -192.48826599121094,
      "logps/rejected": -219.7627716064453,
      "loss": 0.7074,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.16141924262046814,
      "rewards/margins": 0.035948604345321655,
      "rewards/rejected": -0.197367861866951,
      "step": 412
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.978770482432921e-05,
      "logits/chosen": -2.4140584468841553,
      "logits/rejected": -2.394195079803467,
      "logps/chosen": -177.2361297607422,
      "logps/rejected": -201.13475036621094,
      "loss": 0.7109,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.41423746943473816,
      "rewards/margins": 0.01436915248632431,
      "rewards/rejected": -0.42860662937164307,
      "step": 413
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.9786348077216024e-05,
      "logits/chosen": -2.399442672729492,
      "logits/rejected": -2.4223692417144775,
      "logps/chosen": -184.38839721679688,
      "logps/rejected": -176.9226531982422,
      "loss": 0.6572,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2739775776863098,
      "rewards/margins": 0.14736850559711456,
      "rewards/rejected": -0.4213460683822632,
      "step": 414
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.9784987027120236e-05,
      "logits/chosen": -2.331202745437622,
      "logits/rejected": -2.3324198722839355,
      "logps/chosen": -199.08740234375,
      "logps/rejected": -171.50718688964844,
      "loss": 0.7564,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4433014988899231,
      "rewards/margins": -0.06983453035354614,
      "rewards/rejected": -0.37346696853637695,
      "step": 415
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.9783621674278104e-05,
      "logits/chosen": -2.402174949645996,
      "logits/rejected": -2.5383033752441406,
      "logps/chosen": -174.37954711914062,
      "logps/rejected": -165.9345703125,
      "loss": 0.6405,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5206565856933594,
      "rewards/margins": 0.13473454117774963,
      "rewards/rejected": -0.6553912162780762,
      "step": 416
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.978225201892667e-05,
      "logits/chosen": -2.4111886024475098,
      "logits/rejected": -2.3553833961486816,
      "logps/chosen": -220.5755615234375,
      "logps/rejected": -195.2405242919922,
      "loss": 0.7531,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.4778462052345276,
      "rewards/margins": -0.041694507002830505,
      "rewards/rejected": -0.4361516833305359,
      "step": 417
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.97808780613037e-05,
      "logits/chosen": -2.307465076446533,
      "logits/rejected": -2.3099443912506104,
      "logps/chosen": -171.25372314453125,
      "logps/rejected": -200.85305786132812,
      "loss": 0.6326,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.445933997631073,
      "rewards/margins": 0.2173961102962494,
      "rewards/rejected": -0.663330078125,
      "step": 418
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.977949980164773e-05,
      "logits/chosen": -2.3666648864746094,
      "logits/rejected": -2.433722496032715,
      "logps/chosen": -217.6551513671875,
      "logps/rejected": -234.6415557861328,
      "loss": 0.5846,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3358496427536011,
      "rewards/margins": 0.309315949678421,
      "rewards/rejected": -0.6451655626296997,
      "step": 419
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.977811724019802e-05,
      "logits/chosen": -2.336599111557007,
      "logits/rejected": -2.409275770187378,
      "logps/chosen": -191.2588653564453,
      "logps/rejected": -208.126953125,
      "loss": 0.6455,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4646199941635132,
      "rewards/margins": 0.13836784660816193,
      "rewards/rejected": -0.6029877662658691,
      "step": 420
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.9776730377194596e-05,
      "logits/chosen": -2.1639509201049805,
      "logits/rejected": -2.2069835662841797,
      "logps/chosen": -203.13632202148438,
      "logps/rejected": -169.3188018798828,
      "loss": 0.7455,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23145976662635803,
      "rewards/margins": 0.013234168291091919,
      "rewards/rejected": -0.24469394981861115,
      "step": 421
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.9775339212878215e-05,
      "logits/chosen": -2.1670339107513428,
      "logits/rejected": -2.1488430500030518,
      "logps/chosen": -166.50318908691406,
      "logps/rejected": -169.4437713623047,
      "loss": 0.698,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2539255619049072,
      "rewards/margins": 0.030521919950842857,
      "rewards/rejected": -0.28444746136665344,
      "step": 422
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.977394374749039e-05,
      "logits/chosen": -2.336040735244751,
      "logits/rejected": -2.3590757846832275,
      "logps/chosen": -150.48399353027344,
      "logps/rejected": -154.54652404785156,
      "loss": 0.667,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.30479297041893005,
      "rewards/margins": 0.10228873789310455,
      "rewards/rejected": -0.4070816934108734,
      "step": 423
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.9772543981273374e-05,
      "logits/chosen": -2.25268292427063,
      "logits/rejected": -2.267082691192627,
      "logps/chosen": -246.07342529296875,
      "logps/rejected": -230.28424072265625,
      "loss": 0.7204,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.42524397373199463,
      "rewards/margins": -0.008991291746497154,
      "rewards/rejected": -0.4162527322769165,
      "step": 424
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.977113991447017e-05,
      "logits/chosen": -2.35402250289917,
      "logits/rejected": -2.3599345684051514,
      "logps/chosen": -163.17568969726562,
      "logps/rejected": -175.94029235839844,
      "loss": 0.7346,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4037836790084839,
      "rewards/margins": 0.02548382803797722,
      "rewards/rejected": -0.4292675256729126,
      "step": 425
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.976973154732454e-05,
      "logits/chosen": -2.3611769676208496,
      "logits/rejected": -2.3299450874328613,
      "logps/chosen": -205.6555633544922,
      "logps/rejected": -213.8665771484375,
      "loss": 0.7124,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.22228708863258362,
      "rewards/margins": 0.031078480184078217,
      "rewards/rejected": -0.25336551666259766,
      "step": 426
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.976831888008096e-05,
      "logits/chosen": -2.316866636276245,
      "logits/rejected": -2.299907684326172,
      "logps/chosen": -168.08087158203125,
      "logps/rejected": -160.0249786376953,
      "loss": 0.6877,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13129538297653198,
      "rewards/margins": 0.04759233817458153,
      "rewards/rejected": -0.17888770997524261,
      "step": 427
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.976690191298469e-05,
      "logits/chosen": -2.1294949054718018,
      "logits/rejected": -2.1100010871887207,
      "logps/chosen": -176.1876678466797,
      "logps/rejected": -175.78619384765625,
      "loss": 0.7597,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.16827335953712463,
      "rewards/margins": -0.110628642141819,
      "rewards/rejected": -0.05764475464820862,
      "step": 428
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.9765480646281716e-05,
      "logits/chosen": -2.120499849319458,
      "logits/rejected": -2.2186028957366943,
      "logps/chosen": -178.4534912109375,
      "logps/rejected": -200.6439971923828,
      "loss": 0.6143,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11943019181489944,
      "rewards/margins": 0.28324708342552185,
      "rewards/rejected": -0.4026772677898407,
      "step": 429
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.976405508021877e-05,
      "logits/chosen": -2.430680274963379,
      "logits/rejected": -2.5126051902770996,
      "logps/chosen": -201.0321502685547,
      "logps/rejected": -231.42811584472656,
      "loss": 0.7632,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.47548148036003113,
      "rewards/margins": -0.046219632029533386,
      "rewards/rejected": -0.42926183342933655,
      "step": 430
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.9762625215043334e-05,
      "logits/chosen": -2.396533966064453,
      "logits/rejected": -2.4125595092773438,
      "logps/chosen": -236.99049377441406,
      "logps/rejected": -253.93577575683594,
      "loss": 0.7165,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4821574091911316,
      "rewards/margins": -0.007029315456748009,
      "rewards/rejected": -0.4751281142234802,
      "step": 431
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.9761191051003644e-05,
      "logits/chosen": -2.1031394004821777,
      "logits/rejected": -2.143960475921631,
      "logps/chosen": -168.68551635742188,
      "logps/rejected": -209.39537048339844,
      "loss": 0.6219,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1691710501909256,
      "rewards/margins": 0.2633129060268402,
      "rewards/rejected": -0.4324839413166046,
      "step": 432
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.975975258834867e-05,
      "logits/chosen": -2.451735496520996,
      "logits/rejected": -2.410836696624756,
      "logps/chosen": -219.5807342529297,
      "logps/rejected": -238.2914276123047,
      "loss": 0.6939,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.26019036769866943,
      "rewards/margins": 0.09737718105316162,
      "rewards/rejected": -0.35756757855415344,
      "step": 433
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.9758309827328134e-05,
      "logits/chosen": -2.3229422569274902,
      "logits/rejected": -2.3514108657836914,
      "logps/chosen": -181.05836486816406,
      "logps/rejected": -211.49786376953125,
      "loss": 0.7734,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2722405195236206,
      "rewards/margins": -0.07979361712932587,
      "rewards/rejected": -0.19244688749313354,
      "step": 434
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.9756862768192504e-05,
      "logits/chosen": -2.3667490482330322,
      "logits/rejected": -2.328806161880493,
      "logps/chosen": -201.50511169433594,
      "logps/rejected": -179.98643493652344,
      "loss": 0.7353,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.3051126301288605,
      "rewards/margins": -0.007734470069408417,
      "rewards/rejected": -0.29737818241119385,
      "step": 435
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.9755411411192996e-05,
      "logits/chosen": -2.4068377017974854,
      "logits/rejected": -2.532482862472534,
      "logps/chosen": -186.65342712402344,
      "logps/rejected": -205.83154296875,
      "loss": 0.7144,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.17129601538181305,
      "rewards/margins": -0.002290443517267704,
      "rewards/rejected": -0.1690055876970291,
      "step": 436
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.975395575658156e-05,
      "logits/chosen": -2.382935047149658,
      "logits/rejected": -2.4527859687805176,
      "logps/chosen": -200.10215759277344,
      "logps/rejected": -207.49302673339844,
      "loss": 0.7113,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.33162015676498413,
      "rewards/margins": 0.029873518273234367,
      "rewards/rejected": -0.36149367690086365,
      "step": 437
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.9752495804610916e-05,
      "logits/chosen": -2.2459821701049805,
      "logits/rejected": -2.2536497116088867,
      "logps/chosen": -249.44638061523438,
      "logps/rejected": -216.59483337402344,
      "loss": 0.8425,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.3279973268508911,
      "rewards/margins": -0.22765566408634186,
      "rewards/rejected": -0.10034167766571045,
      "step": 438
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.9751031555534504e-05,
      "logits/chosen": -2.416114330291748,
      "logits/rejected": -2.5028560161590576,
      "logps/chosen": -186.53961181640625,
      "logps/rejected": -160.81617736816406,
      "loss": 0.674,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.1654774248600006,
      "rewards/margins": 0.07480952888727188,
      "rewards/rejected": -0.2402869611978531,
      "step": 439
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.9749563009606534e-05,
      "logits/chosen": -2.4489054679870605,
      "logits/rejected": -2.527320623397827,
      "logps/chosen": -202.46507263183594,
      "logps/rejected": -185.72853088378906,
      "loss": 0.6135,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.22387953102588654,
      "rewards/margins": 0.19476865231990814,
      "rewards/rejected": -0.4186481833457947,
      "step": 440
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.9748090167081936e-05,
      "logits/chosen": -2.147719144821167,
      "logits/rejected": -2.249868154525757,
      "logps/chosen": -152.69786071777344,
      "logps/rejected": -190.48863220214844,
      "loss": 0.6568,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.29554668068885803,
      "rewards/margins": 0.11645788699388504,
      "rewards/rejected": -0.4120045602321625,
      "step": 441
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.974661302821641e-05,
      "logits/chosen": -2.142338275909424,
      "logits/rejected": -2.2330148220062256,
      "logps/chosen": -157.3348388671875,
      "logps/rejected": -194.91302490234375,
      "loss": 0.7378,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.13629794120788574,
      "rewards/margins": -0.030040191486477852,
      "rewards/rejected": -0.10625775158405304,
      "step": 442
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.974513159326638e-05,
      "logits/chosen": -2.551227569580078,
      "logits/rejected": -2.5135083198547363,
      "logps/chosen": -201.37612915039062,
      "logps/rejected": -167.43287658691406,
      "loss": 0.6417,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25598540902137756,
      "rewards/margins": 0.18629388511180878,
      "rewards/rejected": -0.4422793388366699,
      "step": 443
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.974364586248904e-05,
      "logits/chosen": -2.174506425857544,
      "logits/rejected": -2.1649577617645264,
      "logps/chosen": -181.4037628173828,
      "logps/rejected": -171.9210968017578,
      "loss": 0.8572,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3619764745235443,
      "rewards/margins": -0.19866357743740082,
      "rewards/rejected": -0.1633128821849823,
      "step": 444
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.974215583614232e-05,
      "logits/chosen": -2.3266444206237793,
      "logits/rejected": -2.418710470199585,
      "logps/chosen": -207.11448669433594,
      "logps/rejected": -287.95574951171875,
      "loss": 0.7347,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.41178515553474426,
      "rewards/margins": -0.002230718731880188,
      "rewards/rejected": -0.40955445170402527,
      "step": 445
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.974066151448488e-05,
      "logits/chosen": -2.5598104000091553,
      "logits/rejected": -2.559662342071533,
      "logps/chosen": -231.59133911132812,
      "logps/rejected": -239.7825469970703,
      "loss": 0.7079,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.42168766260147095,
      "rewards/margins": 0.02927880734205246,
      "rewards/rejected": -0.4509664475917816,
      "step": 446
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.9739162897776146e-05,
      "logits/chosen": -2.3569555282592773,
      "logits/rejected": -2.3967788219451904,
      "logps/chosen": -185.6054229736328,
      "logps/rejected": -192.92169189453125,
      "loss": 0.7437,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4934594929218292,
      "rewards/margins": -0.05449381470680237,
      "rewards/rejected": -0.43896567821502686,
      "step": 447
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.973765998627628e-05,
      "logits/chosen": -2.5715792179107666,
      "logits/rejected": -2.5525949001312256,
      "logps/chosen": -252.23037719726562,
      "logps/rejected": -268.3929443359375,
      "loss": 0.7203,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3784821331501007,
      "rewards/margins": -0.01087496429681778,
      "rewards/rejected": -0.3676071763038635,
      "step": 448
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.973615278024619e-05,
      "logits/chosen": -2.441962718963623,
      "logits/rejected": -2.481163501739502,
      "logps/chosen": -161.66839599609375,
      "logps/rejected": -192.90371704101562,
      "loss": 0.6712,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3044736981391907,
      "rewards/margins": 0.08197857439517975,
      "rewards/rejected": -0.38645222783088684,
      "step": 449
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.9734641279947535e-05,
      "logits/chosen": -2.500675916671753,
      "logits/rejected": -2.487545967102051,
      "logps/chosen": -215.66336059570312,
      "logps/rejected": -232.06967163085938,
      "loss": 0.6796,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4935031235218048,
      "rewards/margins": 0.12285958230495453,
      "rewards/rejected": -0.6163626909255981,
      "step": 450
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.973312548564272e-05,
      "logits/chosen": -2.4314112663269043,
      "logits/rejected": -2.4294910430908203,
      "logps/chosen": -170.31471252441406,
      "logps/rejected": -159.71585083007812,
      "loss": 0.7036,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.32274457812309265,
      "rewards/margins": 0.005533523857593536,
      "rewards/rejected": -0.328278124332428,
      "step": 451
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.9731605397594884e-05,
      "logits/chosen": -2.4075756072998047,
      "logits/rejected": -2.32165789604187,
      "logps/chosen": -186.2813720703125,
      "logps/rejected": -199.41989135742188,
      "loss": 0.655,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3598267138004303,
      "rewards/margins": 0.11941798776388168,
      "rewards/rejected": -0.4792447090148926,
      "step": 452
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.973008101606792e-05,
      "logits/chosen": -2.422717571258545,
      "logits/rejected": -2.5152597427368164,
      "logps/chosen": -173.51361083984375,
      "logps/rejected": -203.63848876953125,
      "loss": 0.7204,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25253090262413025,
      "rewards/margins": 0.029171746224164963,
      "rewards/rejected": -0.2817026376724243,
      "step": 453
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.972855234132646e-05,
      "logits/chosen": -2.517770767211914,
      "logits/rejected": -2.501011848449707,
      "logps/chosen": -202.05300903320312,
      "logps/rejected": -191.65887451171875,
      "loss": 0.7912,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.42360594868659973,
      "rewards/margins": -0.1400626003742218,
      "rewards/rejected": -0.28354334831237793,
      "step": 454
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.9727019373635895e-05,
      "logits/chosen": -2.612367630004883,
      "logits/rejected": -2.617859125137329,
      "logps/chosen": -210.05746459960938,
      "logps/rejected": -199.4877166748047,
      "loss": 0.621,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12031766027212143,
      "rewards/margins": 0.193304181098938,
      "rewards/rejected": -0.3136218190193176,
      "step": 455
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.972548211326235e-05,
      "logits/chosen": -2.5540568828582764,
      "logits/rejected": -2.587779998779297,
      "logps/chosen": -183.94273376464844,
      "logps/rejected": -219.0413818359375,
      "loss": 0.6669,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1985998898744583,
      "rewards/margins": 0.12929418683052063,
      "rewards/rejected": -0.32789406180381775,
      "step": 456
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.9723940560472705e-05,
      "logits/chosen": -2.573051691055298,
      "logits/rejected": -2.6791186332702637,
      "logps/chosen": -193.5257568359375,
      "logps/rejected": -189.68743896484375,
      "loss": 0.7393,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.31839922070503235,
      "rewards/margins": -0.04755366966128349,
      "rewards/rejected": -0.27084553241729736,
      "step": 457
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.972239471553457e-05,
      "logits/chosen": -2.127181053161621,
      "logits/rejected": -2.1366987228393555,
      "logps/chosen": -180.3720703125,
      "logps/rejected": -206.37628173828125,
      "loss": 0.6858,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.14177769422531128,
      "rewards/margins": 0.04955483227968216,
      "rewards/rejected": -0.19133250415325165,
      "step": 458
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.97208445787163e-05,
      "logits/chosen": -2.4841840267181396,
      "logits/rejected": -2.576199531555176,
      "logps/chosen": -175.8242645263672,
      "logps/rejected": -205.9224090576172,
      "loss": 0.7229,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.39802610874176025,
      "rewards/margins": -0.00046849995851516724,
      "rewards/rejected": -0.3975576162338257,
      "step": 459
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.9719290150287026e-05,
      "logits/chosen": -2.4974851608276367,
      "logits/rejected": -2.552859306335449,
      "logps/chosen": -204.64480590820312,
      "logps/rejected": -229.373291015625,
      "loss": 0.7028,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.09486077725887299,
      "rewards/margins": 0.017538849264383316,
      "rewards/rejected": -0.1123996153473854,
      "step": 460
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.9717731430516576e-05,
      "logits/chosen": -2.5486257076263428,
      "logits/rejected": -2.505126953125,
      "logps/chosen": -216.0121307373047,
      "logps/rejected": -204.5452880859375,
      "loss": 0.6996,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1612711399793625,
      "rewards/margins": 0.06856641173362732,
      "rewards/rejected": -0.229837566614151,
      "step": 461
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.9716168419675555e-05,
      "logits/chosen": -2.5866286754608154,
      "logits/rejected": -2.667116403579712,
      "logps/chosen": -198.39340209960938,
      "logps/rejected": -250.51675415039062,
      "loss": 0.6783,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3453162908554077,
      "rewards/margins": 0.07628250867128372,
      "rewards/rejected": -0.42159876227378845,
      "step": 462
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.9714601118035325e-05,
      "logits/chosen": -2.339556932449341,
      "logits/rejected": -2.2843379974365234,
      "logps/chosen": -202.150634765625,
      "logps/rejected": -192.74267578125,
      "loss": 0.674,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1799849569797516,
      "rewards/margins": 0.09412900358438492,
      "rewards/rejected": -0.2741139531135559,
      "step": 463
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.971302952586796e-05,
      "logits/chosen": -2.3514528274536133,
      "logits/rejected": -2.313333749771118,
      "logps/chosen": -216.48309326171875,
      "logps/rejected": -205.1566925048828,
      "loss": 0.7625,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3306673467159271,
      "rewards/margins": -0.07343533635139465,
      "rewards/rejected": -0.25723204016685486,
      "step": 464
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.971145364344628e-05,
      "logits/chosen": -2.475745916366577,
      "logits/rejected": -2.5058016777038574,
      "logps/chosen": -198.23568725585938,
      "logps/rejected": -226.68544006347656,
      "loss": 0.6412,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.38944318890571594,
      "rewards/margins": 0.15158149600028992,
      "rewards/rejected": -0.5410246849060059,
      "step": 465
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.970987347104389e-05,
      "logits/chosen": -2.490628242492676,
      "logits/rejected": -2.5108678340911865,
      "logps/chosen": -180.23779296875,
      "logps/rejected": -232.67515563964844,
      "loss": 0.6863,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3212130665779114,
      "rewards/margins": 0.09368535876274109,
      "rewards/rejected": -0.41489848494529724,
      "step": 466
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.9708289008935096e-05,
      "logits/chosen": -2.6224091053009033,
      "logits/rejected": -2.6361618041992188,
      "logps/chosen": -254.05960083007812,
      "logps/rejected": -247.78634643554688,
      "loss": 0.6909,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.40414828062057495,
      "rewards/margins": 0.051116712391376495,
      "rewards/rejected": -0.4552650451660156,
      "step": 467
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.9706700257394966e-05,
      "logits/chosen": -2.6673390865325928,
      "logits/rejected": -2.6409029960632324,
      "logps/chosen": -160.94146728515625,
      "logps/rejected": -228.09808349609375,
      "loss": 0.7103,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4117045998573303,
      "rewards/margins": 0.047504693269729614,
      "rewards/rejected": -0.45920926332473755,
      "step": 468
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.970510721669932e-05,
      "logits/chosen": -2.12448787689209,
      "logits/rejected": -2.209928512573242,
      "logps/chosen": -247.7496337890625,
      "logps/rejected": -256.54254150390625,
      "loss": 0.602,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.13797114789485931,
      "rewards/margins": 0.25247055292129517,
      "rewards/rejected": -0.3904416561126709,
      "step": 469
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.97035098871247e-05,
      "logits/chosen": -2.531524896621704,
      "logits/rejected": -2.5506911277770996,
      "logps/chosen": -239.89340209960938,
      "logps/rejected": -232.53662109375,
      "loss": 0.6477,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2409316897392273,
      "rewards/margins": 0.11869845539331436,
      "rewards/rejected": -0.35963016748428345,
      "step": 470
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.970190826894842e-05,
      "logits/chosen": -2.4936788082122803,
      "logits/rejected": -2.6132445335388184,
      "logps/chosen": -193.3863067626953,
      "logps/rejected": -196.8754119873047,
      "loss": 0.6012,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33218735456466675,
      "rewards/margins": 0.244198739528656,
      "rewards/rejected": -0.5763860940933228,
      "step": 471
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.9700302362448517e-05,
      "logits/chosen": -2.7358896732330322,
      "logits/rejected": -2.860752582550049,
      "logps/chosen": -193.43789672851562,
      "logps/rejected": -240.47451782226562,
      "loss": 0.6757,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2744450271129608,
      "rewards/margins": 0.07577859610319138,
      "rewards/rejected": -0.3502236306667328,
      "step": 472
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.9698692167903794e-05,
      "logits/chosen": -2.581242799758911,
      "logits/rejected": -2.7139995098114014,
      "logps/chosen": -206.38645935058594,
      "logps/rejected": -210.1461639404297,
      "loss": 0.5954,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3665695786476135,
      "rewards/margins": 0.26593247056007385,
      "rewards/rejected": -0.632502019405365,
      "step": 473
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.9697077685593766e-05,
      "logits/chosen": -2.6083261966705322,
      "logits/rejected": -2.6498053073883057,
      "logps/chosen": -131.37237548828125,
      "logps/rejected": -151.5311737060547,
      "loss": 0.649,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15027746558189392,
      "rewards/margins": 0.126561239361763,
      "rewards/rejected": -0.2768386900424957,
      "step": 474
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.969545891579873e-05,
      "logits/chosen": -2.359971046447754,
      "logits/rejected": -2.3982698917388916,
      "logps/chosen": -157.39181518554688,
      "logps/rejected": -158.3177032470703,
      "loss": 0.6222,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.10072011500597,
      "rewards/margins": 0.24248063564300537,
      "rewards/rejected": -0.34320077300071716,
      "step": 475
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.9693835858799696e-05,
      "logits/chosen": -2.408668279647827,
      "logits/rejected": -2.4540035724639893,
      "logps/chosen": -190.36312866210938,
      "logps/rejected": -204.20359802246094,
      "loss": 0.6997,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.18467962741851807,
      "rewards/margins": 0.015133664011955261,
      "rewards/rejected": -0.19981329143047333,
      "step": 476
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.9692208514878444e-05,
      "logits/chosen": -2.270054340362549,
      "logits/rejected": -2.177553653717041,
      "logps/chosen": -197.311767578125,
      "logps/rejected": -205.99948120117188,
      "loss": 0.7255,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2719980776309967,
      "rewards/margins": -0.018820375204086304,
      "rewards/rejected": -0.2531776428222656,
      "step": 477
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.969057688431748e-05,
      "logits/chosen": -2.4876270294189453,
      "logits/rejected": -2.516055107116699,
      "logps/chosen": -150.5498046875,
      "logps/rejected": -173.20225524902344,
      "loss": 0.6482,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2806074023246765,
      "rewards/margins": 0.18431711196899414,
      "rewards/rejected": -0.46492451429367065,
      "step": 478
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.968894096740006e-05,
      "logits/chosen": -2.699005365371704,
      "logits/rejected": -2.706855297088623,
      "logps/chosen": -208.8563232421875,
      "logps/rejected": -229.45355224609375,
      "loss": 0.6569,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3221612572669983,
      "rewards/margins": 0.1469050496816635,
      "rewards/rejected": -0.4690663516521454,
      "step": 479
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.968730076441017e-05,
      "logits/chosen": -2.752363681793213,
      "logits/rejected": -2.8258612155914307,
      "logps/chosen": -233.5675811767578,
      "logps/rejected": -261.7062683105469,
      "loss": 0.7128,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.604475200176239,
      "rewards/margins": 0.022365085780620575,
      "rewards/rejected": -0.6268402338027954,
      "step": 480
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.9685656275632575e-05,
      "logits/chosen": -2.5177550315856934,
      "logits/rejected": -2.510917901992798,
      "logps/chosen": -217.02195739746094,
      "logps/rejected": -189.55174255371094,
      "loss": 0.6434,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3391067683696747,
      "rewards/margins": 0.13497701287269592,
      "rewards/rejected": -0.4740837812423706,
      "step": 481
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.968400750135276e-05,
      "logits/chosen": -2.540860652923584,
      "logits/rejected": -2.582563877105713,
      "logps/chosen": -194.17721557617188,
      "logps/rejected": -198.79684448242188,
      "loss": 0.6679,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.574460506439209,
      "rewards/margins": 0.10229263454675674,
      "rewards/rejected": -0.6767531633377075,
      "step": 482
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.968235444185695e-05,
      "logits/chosen": -2.7020184993743896,
      "logits/rejected": -2.7175564765930176,
      "logps/chosen": -211.08863830566406,
      "logps/rejected": -235.41836547851562,
      "loss": 0.7235,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5369959473609924,
      "rewards/margins": 0.011504769325256348,
      "rewards/rejected": -0.5485007762908936,
      "step": 483
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.968069709743212e-05,
      "logits/chosen": -2.4362404346466064,
      "logits/rejected": -2.5247411727905273,
      "logps/chosen": -149.7013397216797,
      "logps/rejected": -185.23251342773438,
      "loss": 0.5556,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.1017133966088295,
      "rewards/margins": 0.3292151391506195,
      "rewards/rejected": -0.4309285283088684,
      "step": 484
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.9679035468365986e-05,
      "logits/chosen": -2.3711907863616943,
      "logits/rejected": -2.412506580352783,
      "logps/chosen": -202.7202911376953,
      "logps/rejected": -219.67977905273438,
      "loss": 0.6658,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3317970931529999,
      "rewards/margins": 0.12470696866512299,
      "rewards/rejected": -0.45650404691696167,
      "step": 485
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.967736955494703e-05,
      "logits/chosen": -2.204392910003662,
      "logits/rejected": -2.2046332359313965,
      "logps/chosen": -195.41537475585938,
      "logps/rejected": -196.8472442626953,
      "loss": 0.7358,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.43655478954315186,
      "rewards/margins": -0.000651337206363678,
      "rewards/rejected": -0.4359034597873688,
      "step": 486
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.9675699357464445e-05,
      "logits/chosen": -2.198075771331787,
      "logits/rejected": -2.1894662380218506,
      "logps/chosen": -248.62149047851562,
      "logps/rejected": -267.9733581542969,
      "loss": 0.5947,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14261871576309204,
      "rewards/margins": 0.31586068868637085,
      "rewards/rejected": -0.4584794044494629,
      "step": 487
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.967402487620818e-05,
      "logits/chosen": -2.5536584854125977,
      "logits/rejected": -2.577322006225586,
      "logps/chosen": -224.8876190185547,
      "logps/rejected": -227.71383666992188,
      "loss": 0.663,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.40421125292778015,
      "rewards/margins": 0.11385571211576462,
      "rewards/rejected": -0.5180670022964478,
      "step": 488
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.9672346111468934e-05,
      "logits/chosen": -2.508915424346924,
      "logits/rejected": -2.624011993408203,
      "logps/chosen": -181.42379760742188,
      "logps/rejected": -231.30458068847656,
      "loss": 0.6309,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.31741198897361755,
      "rewards/margins": 0.24159038066864014,
      "rewards/rejected": -0.5590023994445801,
      "step": 489
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.967066306353816e-05,
      "logits/chosen": -2.4504952430725098,
      "logits/rejected": -2.4388234615325928,
      "logps/chosen": -229.2549591064453,
      "logps/rejected": -238.2850341796875,
      "loss": 0.6922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.47322726249694824,
      "rewards/margins": 0.07009989768266678,
      "rewards/rejected": -0.5433271527290344,
      "step": 490
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.966897573270801e-05,
      "logits/chosen": -2.555962562561035,
      "logits/rejected": -2.5629987716674805,
      "logps/chosen": -172.14407348632812,
      "logps/rejected": -168.1295166015625,
      "loss": 0.7069,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2661295533180237,
      "rewards/margins": 0.07032840698957443,
      "rewards/rejected": -0.3364579379558563,
      "step": 491
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.966728411927144e-05,
      "logits/chosen": -2.4634742736816406,
      "logits/rejected": -2.4717013835906982,
      "logps/chosen": -206.756591796875,
      "logps/rejected": -202.884033203125,
      "loss": 0.7013,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5119624733924866,
      "rewards/margins": 0.043263014405965805,
      "rewards/rejected": -0.5552253723144531,
      "step": 492
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.9665588223522096e-05,
      "logits/chosen": -2.333207607269287,
      "logits/rejected": -2.3922958374023438,
      "logps/chosen": -210.04153442382812,
      "logps/rejected": -239.76979064941406,
      "loss": 0.7059,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.32687515020370483,
      "rewards/margins": 0.09048546105623245,
      "rewards/rejected": -0.41736066341400146,
      "step": 493
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.96638880457544e-05,
      "logits/chosen": -2.414531707763672,
      "logits/rejected": -2.4198668003082275,
      "logps/chosen": -225.68203735351562,
      "logps/rejected": -252.04661560058594,
      "loss": 0.6259,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44954052567481995,
      "rewards/margins": 0.19488830864429474,
      "rewards/rejected": -0.6444287896156311,
      "step": 494
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.9662183586263514e-05,
      "logits/chosen": -2.523651361465454,
      "logits/rejected": -2.4278194904327393,
      "logps/chosen": -222.92274475097656,
      "logps/rejected": -203.31668090820312,
      "loss": 0.8255,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6954134106636047,
      "rewards/margins": -0.12540754675865173,
      "rewards/rejected": -0.5700058937072754,
      "step": 495
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.966047484534533e-05,
      "logits/chosen": -2.465467929840088,
      "logits/rejected": -2.4773805141448975,
      "logps/chosen": -215.38780212402344,
      "logps/rejected": -222.23593139648438,
      "loss": 0.6348,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.43675118684768677,
      "rewards/margins": 0.19925275444984436,
      "rewards/rejected": -0.6360039710998535,
      "step": 496
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.965876182329648e-05,
      "logits/chosen": -2.4241979122161865,
      "logits/rejected": -2.4904754161834717,
      "logps/chosen": -176.40281677246094,
      "logps/rejected": -189.89398193359375,
      "loss": 0.7316,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.35698071122169495,
      "rewards/margins": 0.12830819189548492,
      "rewards/rejected": -0.48528894782066345,
      "step": 497
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.965704452041437e-05,
      "logits/chosen": -2.4424967765808105,
      "logits/rejected": -2.447554588317871,
      "logps/chosen": -186.7346954345703,
      "logps/rejected": -202.56729125976562,
      "loss": 0.6988,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.379626601934433,
      "rewards/margins": 0.05914265662431717,
      "rewards/rejected": -0.43876922130584717,
      "step": 498
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.9655322936997115e-05,
      "logits/chosen": -2.384314775466919,
      "logits/rejected": -2.6753270626068115,
      "logps/chosen": -168.5001220703125,
      "logps/rejected": -201.58462524414062,
      "loss": 0.5711,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.30074769258499146,
      "rewards/margins": 0.31708773970603943,
      "rewards/rejected": -0.6178354024887085,
      "step": 499
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.9653597073343594e-05,
      "logits/chosen": -2.44107723236084,
      "logits/rejected": -2.518571615219116,
      "logps/chosen": -190.8749542236328,
      "logps/rejected": -195.59471130371094,
      "loss": 0.6465,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3541903495788574,
      "rewards/margins": 0.2186165750026703,
      "rewards/rejected": -0.5728069543838501,
      "step": 500
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.9651866929753424e-05,
      "logits/chosen": -2.580932140350342,
      "logits/rejected": -2.6375184059143066,
      "logps/chosen": -201.61231994628906,
      "logps/rejected": -195.1832275390625,
      "loss": 0.6788,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4448980689048767,
      "rewards/margins": 0.1541019082069397,
      "rewards/rejected": -0.5990000367164612,
      "step": 501
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.9650132506526944e-05,
      "logits/chosen": -2.601297378540039,
      "logits/rejected": -2.5584542751312256,
      "logps/chosen": -188.46102905273438,
      "logps/rejected": -176.1941680908203,
      "loss": 0.6884,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.32628124952316284,
      "rewards/margins": 0.053992465138435364,
      "rewards/rejected": -0.3802737295627594,
      "step": 502
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.9648393803965284e-05,
      "logits/chosen": -2.594449281692505,
      "logits/rejected": -2.6243650913238525,
      "logps/chosen": -180.65838623046875,
      "logps/rejected": -173.06399536132812,
      "loss": 0.608,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3709149658679962,
      "rewards/margins": 0.24527978897094727,
      "rewards/rejected": -0.6161947846412659,
      "step": 503
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.9646650822370265e-05,
      "logits/chosen": -2.623140811920166,
      "logits/rejected": -2.6087071895599365,
      "logps/chosen": -175.10757446289062,
      "logps/rejected": -198.5271759033203,
      "loss": 0.6623,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34849774837493896,
      "rewards/margins": 0.2720443308353424,
      "rewards/rejected": -0.6205421090126038,
      "step": 504
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.964490356204449e-05,
      "logits/chosen": -2.3962643146514893,
      "logits/rejected": -2.4296982288360596,
      "logps/chosen": -171.71841430664062,
      "logps/rejected": -189.31715393066406,
      "loss": 0.7731,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5484548211097717,
      "rewards/margins": 0.05871535837650299,
      "rewards/rejected": -0.6071702241897583,
      "step": 505
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.964315202329127e-05,
      "logits/chosen": -2.1054232120513916,
      "logits/rejected": -2.0613820552825928,
      "logps/chosen": -196.4398956298828,
      "logps/rejected": -197.6771240234375,
      "loss": 0.6993,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4299621880054474,
      "rewards/margins": 0.10146878659725189,
      "rewards/rejected": -0.5314310193061829,
      "step": 506
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.96413962064147e-05,
      "logits/chosen": -2.4437365531921387,
      "logits/rejected": -2.6009879112243652,
      "logps/chosen": -218.52403259277344,
      "logps/rejected": -253.1461639404297,
      "loss": 0.5733,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.4768885374069214,
      "rewards/margins": 0.29149848222732544,
      "rewards/rejected": -0.7683870196342468,
      "step": 507
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.963963611171957e-05,
      "logits/chosen": -2.737818479537964,
      "logits/rejected": -2.8296992778778076,
      "logps/chosen": -194.77084350585938,
      "logps/rejected": -196.74485778808594,
      "loss": 0.6388,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4071928560733795,
      "rewards/margins": 0.17611998319625854,
      "rewards/rejected": -0.5833128690719604,
      "step": 508
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.9637871739511454e-05,
      "logits/chosen": -2.5671932697296143,
      "logits/rejected": -2.4917585849761963,
      "logps/chosen": -208.9228057861328,
      "logps/rejected": -204.2896270751953,
      "loss": 0.7151,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.38905590772628784,
      "rewards/margins": 0.02341422438621521,
      "rewards/rejected": -0.41247016191482544,
      "step": 509
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.963610309009665e-05,
      "logits/chosen": -2.3518598079681396,
      "logits/rejected": -2.3721611499786377,
      "logps/chosen": -154.779296875,
      "logps/rejected": -190.75613403320312,
      "loss": 0.6552,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.35764750838279724,
      "rewards/margins": 0.1814635694026947,
      "rewards/rejected": -0.5391110777854919,
      "step": 510
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.9634330163782205e-05,
      "logits/chosen": -2.432107448577881,
      "logits/rejected": -2.495102882385254,
      "logps/chosen": -248.51341247558594,
      "logps/rejected": -233.74266052246094,
      "loss": 0.7034,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.45434871315956116,
      "rewards/margins": 0.08803940564393997,
      "rewards/rejected": -0.5423881411552429,
      "step": 511
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.963255296087589e-05,
      "logits/chosen": -2.4448506832122803,
      "logits/rejected": -2.462013006210327,
      "logps/chosen": -143.73773193359375,
      "logps/rejected": -182.57113647460938,
      "loss": 0.6068,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3137112855911255,
      "rewards/margins": 0.21627672016620636,
      "rewards/rejected": -0.5299879908561707,
      "step": 512
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.9630771481686244e-05,
      "logits/chosen": -2.4147915840148926,
      "logits/rejected": -2.368267297744751,
      "logps/chosen": -189.82333374023438,
      "logps/rejected": -175.15501403808594,
      "loss": 0.6099,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34441184997558594,
      "rewards/margins": 0.29255932569503784,
      "rewards/rejected": -0.636971116065979,
      "step": 513
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.9628985726522535e-05,
      "logits/chosen": -2.424818992614746,
      "logits/rejected": -2.4481663703918457,
      "logps/chosen": -204.38375854492188,
      "logps/rejected": -204.50376892089844,
      "loss": 0.6846,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4307776689529419,
      "rewards/margins": 0.06390251964330673,
      "rewards/rejected": -0.4946801960468292,
      "step": 514
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.9627195695694774e-05,
      "logits/chosen": -2.523521661758423,
      "logits/rejected": -2.5060675144195557,
      "logps/chosen": -163.70565795898438,
      "logps/rejected": -207.99964904785156,
      "loss": 0.6256,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.32005366683006287,
      "rewards/margins": 0.24324071407318115,
      "rewards/rejected": -0.5632944107055664,
      "step": 515
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.9625401389513715e-05,
      "logits/chosen": -2.4181556701660156,
      "logits/rejected": -2.369680881500244,
      "logps/chosen": -205.77340698242188,
      "logps/rejected": -155.3488311767578,
      "loss": 0.8924,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.7356570363044739,
      "rewards/margins": -0.26000097393989563,
      "rewards/rejected": -0.47565609216690063,
      "step": 516
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.9623602808290855e-05,
      "logits/chosen": -2.544684410095215,
      "logits/rejected": -2.5888850688934326,
      "logps/chosen": -229.81246948242188,
      "logps/rejected": -251.07574462890625,
      "loss": 0.6031,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4079195559024811,
      "rewards/margins": 0.29860368371009827,
      "rewards/rejected": -0.7065231800079346,
      "step": 517
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.962179995233843e-05,
      "logits/chosen": -2.397979259490967,
      "logits/rejected": -2.5106217861175537,
      "logps/chosen": -157.80418395996094,
      "logps/rejected": -175.9936065673828,
      "loss": 0.6093,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.44883906841278076,
      "rewards/margins": 0.32764846086502075,
      "rewards/rejected": -0.7764875888824463,
      "step": 518
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.961999282196943e-05,
      "logits/chosen": -2.4970529079437256,
      "logits/rejected": -2.582210063934326,
      "logps/chosen": -194.96694946289062,
      "logps/rejected": -198.24681091308594,
      "loss": 0.6883,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.37289613485336304,
      "rewards/margins": 0.0580718033015728,
      "rewards/rejected": -0.4309679865837097,
      "step": 519
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.9618181417497566e-05,
      "logits/chosen": -2.469606399536133,
      "logits/rejected": -2.54701566696167,
      "logps/chosen": -215.56979370117188,
      "logps/rejected": -233.58209228515625,
      "loss": 0.691,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5863683819770813,
      "rewards/margins": 0.03697293624281883,
      "rewards/rejected": -0.6233413815498352,
      "step": 520
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.961636573923732e-05,
      "logits/chosen": -2.4047884941101074,
      "logits/rejected": -2.414797782897949,
      "logps/chosen": -226.51625061035156,
      "logps/rejected": -212.8037872314453,
      "loss": 0.6559,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4074936807155609,
      "rewards/margins": 0.16430170834064484,
      "rewards/rejected": -0.5717953443527222,
      "step": 521
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.9614545787503886e-05,
      "logits/chosen": -2.4261462688446045,
      "logits/rejected": -2.482429027557373,
      "logps/chosen": -197.58526611328125,
      "logps/rejected": -215.66322326660156,
      "loss": 0.6091,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4302670359611511,
      "rewards/margins": 0.2678428888320923,
      "rewards/rejected": -0.6981098651885986,
      "step": 522
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.961272156261321e-05,
      "logits/chosen": -2.22928786277771,
      "logits/rejected": -2.3709988594055176,
      "logps/chosen": -157.08120727539062,
      "logps/rejected": -186.32533264160156,
      "loss": 0.5613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32108116149902344,
      "rewards/margins": 0.476359486579895,
      "rewards/rejected": -0.7974405884742737,
      "step": 523
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.961089306488199e-05,
      "logits/chosen": -2.3764328956604004,
      "logits/rejected": -2.423450231552124,
      "logps/chosen": -225.89686584472656,
      "logps/rejected": -250.8079376220703,
      "loss": 0.6351,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4355972707271576,
      "rewards/margins": 0.2791078984737396,
      "rewards/rejected": -0.7147052884101868,
      "step": 524
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.960906029462766e-05,
      "logits/chosen": -2.338782548904419,
      "logits/rejected": -2.423982858657837,
      "logps/chosen": -210.87510681152344,
      "logps/rejected": -207.5828857421875,
      "loss": 0.7054,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5041165351867676,
      "rewards/margins": 0.092525914311409,
      "rewards/rejected": -0.5966423749923706,
      "step": 525
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.96072232521684e-05,
      "logits/chosen": -2.4865126609802246,
      "logits/rejected": -2.55895733833313,
      "logps/chosen": -290.5992431640625,
      "logps/rejected": -275.2317810058594,
      "loss": 0.6608,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5548466444015503,
      "rewards/margins": 0.17232416570186615,
      "rewards/rejected": -0.7271708250045776,
      "step": 526
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.96053819378231e-05,
      "logits/chosen": -2.6567628383636475,
      "logits/rejected": -2.5933961868286133,
      "logps/chosen": -183.54367065429688,
      "logps/rejected": -175.34194946289062,
      "loss": 0.7362,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.7040901780128479,
      "rewards/margins": -0.04544142633676529,
      "rewards/rejected": -0.6586487889289856,
      "step": 527
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.960353635191145e-05,
      "logits/chosen": -2.514308452606201,
      "logits/rejected": -2.5751612186431885,
      "logps/chosen": -171.15457153320312,
      "logps/rejected": -163.67311096191406,
      "loss": 0.606,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.340698778629303,
      "rewards/margins": 0.42078471183776855,
      "rewards/rejected": -0.7614834904670715,
      "step": 528
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.9601686494753826e-05,
      "logits/chosen": -2.318474292755127,
      "logits/rejected": -2.2866580486297607,
      "logps/chosen": -155.76071166992188,
      "logps/rejected": -156.44679260253906,
      "loss": 0.7173,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.30408087372779846,
      "rewards/margins": -0.002507832832634449,
      "rewards/rejected": -0.3015730381011963,
      "step": 529
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.959983236667138e-05,
      "logits/chosen": -2.577028274536133,
      "logits/rejected": -2.6590182781219482,
      "logps/chosen": -142.8551483154297,
      "logps/rejected": -166.59149169921875,
      "loss": 0.6867,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5840120315551758,
      "rewards/margins": 0.07422871142625809,
      "rewards/rejected": -0.6582406759262085,
      "step": 530
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.959797396798599e-05,
      "logits/chosen": -2.409472703933716,
      "logits/rejected": -2.4960474967956543,
      "logps/chosen": -150.38006591796875,
      "logps/rejected": -176.5684814453125,
      "loss": 0.6631,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5444418787956238,
      "rewards/margins": 0.12680965662002563,
      "rewards/rejected": -0.6712515354156494,
      "step": 531
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.9596111299020284e-05,
      "logits/chosen": -2.336719512939453,
      "logits/rejected": -2.3440167903900146,
      "logps/chosen": -184.50473022460938,
      "logps/rejected": -187.92713928222656,
      "loss": 0.627,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.33718907833099365,
      "rewards/margins": 0.19500240683555603,
      "rewards/rejected": -0.5321914553642273,
      "step": 532
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.959424436009762e-05,
      "logits/chosen": -2.1330745220184326,
      "logits/rejected": -2.2025442123413086,
      "logps/chosen": -255.2789306640625,
      "logps/rejected": -287.85736083984375,
      "loss": 0.5781,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6389483213424683,
      "rewards/margins": 0.5605921149253845,
      "rewards/rejected": -1.199540376663208,
      "step": 533
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.9592373151542105e-05,
      "logits/chosen": -2.783942699432373,
      "logits/rejected": -2.766120672225952,
      "logps/chosen": -185.41404724121094,
      "logps/rejected": -203.95669555664062,
      "loss": 0.7158,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4609447717666626,
      "rewards/margins": 0.003200806677341461,
      "rewards/rejected": -0.46414560079574585,
      "step": 534
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.959049767367859e-05,
      "logits/chosen": -2.4105114936828613,
      "logits/rejected": -2.6107630729675293,
      "logps/chosen": -172.59347534179688,
      "logps/rejected": -212.30836486816406,
      "loss": 0.6269,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5956838726997375,
      "rewards/margins": 0.2615227699279785,
      "rewards/rejected": -0.8572065830230713,
      "step": 535
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.958861792683266e-05,
      "logits/chosen": -2.1016757488250732,
      "logits/rejected": -2.1774649620056152,
      "logps/chosen": -204.3402862548828,
      "logps/rejected": -215.18890380859375,
      "loss": 0.6164,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.49696028232574463,
      "rewards/margins": 0.2144584059715271,
      "rewards/rejected": -0.7114187479019165,
      "step": 536
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.958673391133065e-05,
      "logits/chosen": -2.6260104179382324,
      "logits/rejected": -2.648331880569458,
      "logps/chosen": -180.17063903808594,
      "logps/rejected": -175.27981567382812,
      "loss": 0.7315,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7630718946456909,
      "rewards/margins": 0.05031493306159973,
      "rewards/rejected": -0.813386857509613,
      "step": 537
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.958484562749963e-05,
      "logits/chosen": -2.3528480529785156,
      "logits/rejected": -2.35026216506958,
      "logps/chosen": -240.7522735595703,
      "logps/rejected": -284.01177978515625,
      "loss": 0.6558,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5610861778259277,
      "rewards/margins": 0.1676333099603653,
      "rewards/rejected": -0.7287194132804871,
      "step": 538
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.9582953075667406e-05,
      "logits/chosen": -2.7893500328063965,
      "logits/rejected": -2.695694923400879,
      "logps/chosen": -213.7672119140625,
      "logps/rejected": -197.13304138183594,
      "loss": 0.7034,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7025653719902039,
      "rewards/margins": 0.08315228670835495,
      "rewards/rejected": -0.7857176661491394,
      "step": 539
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.958105625616253e-05,
      "logits/chosen": -2.6460516452789307,
      "logits/rejected": -2.6226911544799805,
      "logps/chosen": -186.30870056152344,
      "logps/rejected": -169.23074340820312,
      "loss": 0.684,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4503140449523926,
      "rewards/margins": 0.0906258150935173,
      "rewards/rejected": -0.5409399271011353,
      "step": 540
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.95791551693143e-05,
      "logits/chosen": -2.6068363189697266,
      "logits/rejected": -2.645115852355957,
      "logps/chosen": -225.66354370117188,
      "logps/rejected": -248.78900146484375,
      "loss": 0.7087,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.7523027062416077,
      "rewards/margins": 0.06183048337697983,
      "rewards/rejected": -0.8141331672668457,
      "step": 541
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.957724981545275e-05,
      "logits/chosen": -2.472942352294922,
      "logits/rejected": -2.5449466705322266,
      "logps/chosen": -165.9869384765625,
      "logps/rejected": -195.7375946044922,
      "loss": 0.6983,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5627427697181702,
      "rewards/margins": 0.17754918336868286,
      "rewards/rejected": -0.7402920126914978,
      "step": 542
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.957534019490865e-05,
      "logits/chosen": -2.6267778873443604,
      "logits/rejected": -2.8627214431762695,
      "logps/chosen": -208.47406005859375,
      "logps/rejected": -230.64390563964844,
      "loss": 0.5822,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6202707290649414,
      "rewards/margins": 0.35299497842788696,
      "rewards/rejected": -0.9732657074928284,
      "step": 543
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.957342630801353e-05,
      "logits/chosen": -2.302680730819702,
      "logits/rejected": -2.4394922256469727,
      "logps/chosen": -169.56790161132812,
      "logps/rejected": -198.37738037109375,
      "loss": 0.6269,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44470933079719543,
      "rewards/margins": 0.24785885214805603,
      "rewards/rejected": -0.6925681233406067,
      "step": 544
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.957150815509963e-05,
      "logits/chosen": -2.454188823699951,
      "logits/rejected": -2.548893451690674,
      "logps/chosen": -170.51040649414062,
      "logps/rejected": -232.72727966308594,
      "loss": 0.7024,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3639673590660095,
      "rewards/margins": 0.021631799638271332,
      "rewards/rejected": -0.38559919595718384,
      "step": 545
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.9569585736499945e-05,
      "logits/chosen": -2.337388515472412,
      "logits/rejected": -2.4017117023468018,
      "logps/chosen": -177.1454620361328,
      "logps/rejected": -191.3271484375,
      "loss": 0.5661,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3925960958003998,
      "rewards/margins": 0.34767282009124756,
      "rewards/rejected": -0.740268886089325,
      "step": 546
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.9567659052548234e-05,
      "logits/chosen": -2.653782367706299,
      "logits/rejected": -2.6659772396087646,
      "logps/chosen": -198.43539428710938,
      "logps/rejected": -176.4965362548828,
      "loss": 0.7005,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5790706872940063,
      "rewards/margins": 0.11143729835748672,
      "rewards/rejected": -0.6905080080032349,
      "step": 547
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.956572810357896e-05,
      "logits/chosen": -2.5503158569335938,
      "logits/rejected": -2.520476818084717,
      "logps/chosen": -190.98855590820312,
      "logps/rejected": -173.04501342773438,
      "loss": 0.7238,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5375619530677795,
      "rewards/margins": 0.01934710703790188,
      "rewards/rejected": -0.5569091439247131,
      "step": 548
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.956379288992733e-05,
      "logits/chosen": -2.2868709564208984,
      "logits/rejected": -2.358032464981079,
      "logps/chosen": -163.09713745117188,
      "logps/rejected": -187.16030883789062,
      "loss": 0.5837,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3862106502056122,
      "rewards/margins": 0.4660491943359375,
      "rewards/rejected": -0.8522598147392273,
      "step": 549
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.956185341192933e-05,
      "logits/chosen": -2.522181987762451,
      "logits/rejected": -2.5097920894622803,
      "logps/chosen": -167.14633178710938,
      "logps/rejected": -205.06365966796875,
      "loss": 0.7648,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5081343650817871,
      "rewards/margins": -0.012615025043487549,
      "rewards/rejected": -0.4955193102359772,
      "step": 550
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.9559909669921635e-05,
      "logits/chosen": -2.3636226654052734,
      "logits/rejected": -2.438326120376587,
      "logps/chosen": -197.59310913085938,
      "logps/rejected": -186.26112365722656,
      "loss": 0.7066,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5027388334274292,
      "rewards/margins": 0.07569919526576996,
      "rewards/rejected": -0.5784379839897156,
      "step": 551
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.9557961664241694e-05,
      "logits/chosen": -2.425126552581787,
      "logits/rejected": -2.5314579010009766,
      "logps/chosen": -181.5507049560547,
      "logps/rejected": -211.9708251953125,
      "loss": 0.7133,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6780929565429688,
      "rewards/margins": 0.19112253189086914,
      "rewards/rejected": -0.8692154884338379,
      "step": 552
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.955600939522769e-05,
      "logits/chosen": -2.52880597114563,
      "logits/rejected": -2.5222525596618652,
      "logps/chosen": -178.28521728515625,
      "logps/rejected": -210.90565490722656,
      "loss": 0.587,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5996692180633545,
      "rewards/margins": 0.25250738859176636,
      "rewards/rejected": -0.8521764874458313,
      "step": 553
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.955405286321854e-05,
      "logits/chosen": -2.3732471466064453,
      "logits/rejected": -2.4644832611083984,
      "logps/chosen": -210.51051330566406,
      "logps/rejected": -208.96826171875,
      "loss": 0.5982,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5088058114051819,
      "rewards/margins": 0.2920345067977905,
      "rewards/rejected": -0.8008402585983276,
      "step": 554
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.95520920685539e-05,
      "logits/chosen": -2.4812893867492676,
      "logits/rejected": -2.5562007427215576,
      "logps/chosen": -170.83322143554688,
      "logps/rejected": -192.9698486328125,
      "loss": 0.5716,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6304159164428711,
      "rewards/margins": 0.37207356095314026,
      "rewards/rejected": -1.002489447593689,
      "step": 555
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.9550127011574176e-05,
      "logits/chosen": -2.281599760055542,
      "logits/rejected": -2.3443262577056885,
      "logps/chosen": -188.46249389648438,
      "logps/rejected": -179.5142364501953,
      "loss": 0.6338,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.41307827830314636,
      "rewards/margins": 0.17765894532203674,
      "rewards/rejected": -0.5907372236251831,
      "step": 556
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.95481576926205e-05,
      "logits/chosen": -2.584645986557007,
      "logits/rejected": -2.604243755340576,
      "logps/chosen": -181.7239990234375,
      "logps/rejected": -209.90145874023438,
      "loss": 0.5821,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7002069354057312,
      "rewards/margins": 0.43418359756469727,
      "rewards/rejected": -1.1343904733657837,
      "step": 557
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.954618411203476e-05,
      "logits/chosen": -2.679819107055664,
      "logits/rejected": -2.7520828247070312,
      "logps/chosen": -217.54901123046875,
      "logps/rejected": -236.81210327148438,
      "loss": 0.6187,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.734368085861206,
      "rewards/margins": 0.29621487855911255,
      "rewards/rejected": -1.0305829048156738,
      "step": 558
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.954420627015957e-05,
      "logits/chosen": -2.530925750732422,
      "logits/rejected": -2.5949485301971436,
      "logps/chosen": -190.8072509765625,
      "logps/rejected": -226.07455444335938,
      "loss": 0.5481,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5543414950370789,
      "rewards/margins": 0.42910295724868774,
      "rewards/rejected": -0.9834444522857666,
      "step": 559
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.954222416733829e-05,
      "logits/chosen": -2.4444892406463623,
      "logits/rejected": -2.4189417362213135,
      "logps/chosen": -275.56109619140625,
      "logps/rejected": -248.05233764648438,
      "loss": 0.7178,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5780796408653259,
      "rewards/margins": 0.0615503191947937,
      "rewards/rejected": -0.6396299600601196,
      "step": 560
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.954023780391501e-05,
      "logits/chosen": -2.360807180404663,
      "logits/rejected": -2.308104991912842,
      "logps/chosen": -172.66574096679688,
      "logps/rejected": -191.79412841796875,
      "loss": 0.5558,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.709125816822052,
      "rewards/margins": 0.38629868626594543,
      "rewards/rejected": -1.0954244136810303,
      "step": 561
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.953824718023459e-05,
      "logits/chosen": -2.140808582305908,
      "logits/rejected": -2.2450931072235107,
      "logps/chosen": -135.81964111328125,
      "logps/rejected": -176.7286376953125,
      "loss": 0.7815,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7152988910675049,
      "rewards/margins": 0.006897151470184326,
      "rewards/rejected": -0.722196102142334,
      "step": 562
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.953625229664259e-05,
      "logits/chosen": -2.2745089530944824,
      "logits/rejected": -2.2524397373199463,
      "logps/chosen": -174.69879150390625,
      "logps/rejected": -193.57516479492188,
      "loss": 0.8268,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6095337867736816,
      "rewards/margins": -0.15611900389194489,
      "rewards/rejected": -0.45341476798057556,
      "step": 563
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.953425315348534e-05,
      "logits/chosen": -2.4158096313476562,
      "logits/rejected": -2.5616989135742188,
      "logps/chosen": -161.41812133789062,
      "logps/rejected": -195.5721893310547,
      "loss": 0.5776,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4711029529571533,
      "rewards/margins": 0.4865836799144745,
      "rewards/rejected": -0.957686722278595,
      "step": 564
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.953224975110988e-05,
      "logits/chosen": -2.3829874992370605,
      "logits/rejected": -2.3937342166900635,
      "logps/chosen": -168.71534729003906,
      "logps/rejected": -178.2794189453125,
      "loss": 0.7187,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5582124590873718,
      "rewards/margins": -0.010660316795110703,
      "rewards/rejected": -0.5475521087646484,
      "step": 565
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.9530242089864026e-05,
      "logits/chosen": -2.3829197883605957,
      "logits/rejected": -2.462663173675537,
      "logps/chosen": -173.0697021484375,
      "logps/rejected": -207.88348388671875,
      "loss": 0.6547,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4435291290283203,
      "rewards/margins": 0.13191349804401398,
      "rewards/rejected": -0.5754426717758179,
      "step": 566
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.9528230170096305e-05,
      "logits/chosen": -2.319032907485962,
      "logits/rejected": -2.3326690196990967,
      "logps/chosen": -169.28501892089844,
      "logps/rejected": -194.19927978515625,
      "loss": 0.7851,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7481305599212646,
      "rewards/margins": 0.09495264291763306,
      "rewards/rejected": -0.8430831432342529,
      "step": 567
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.952621399215598e-05,
      "logits/chosen": -2.630171537399292,
      "logits/rejected": -2.617821455001831,
      "logps/chosen": -192.14907836914062,
      "logps/rejected": -193.33782958984375,
      "loss": 0.687,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.47104740142822266,
      "rewards/margins": 0.11374848335981369,
      "rewards/rejected": -0.5847958922386169,
      "step": 568
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.9524193556393083e-05,
      "logits/chosen": -2.3562679290771484,
      "logits/rejected": -2.388498306274414,
      "logps/chosen": -212.62057495117188,
      "logps/rejected": -230.94476318359375,
      "loss": 0.6921,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.575402557849884,
      "rewards/margins": 0.19730643928050995,
      "rewards/rejected": -0.7727090120315552,
      "step": 569
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.952216886315837e-05,
      "logits/chosen": -2.414254903793335,
      "logits/rejected": -2.4329581260681152,
      "logps/chosen": -175.03147888183594,
      "logps/rejected": -214.68109130859375,
      "loss": 0.6274,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6071965098381042,
      "rewards/margins": 0.29113632440567017,
      "rewards/rejected": -0.8983328342437744,
      "step": 570
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.952013991280332e-05,
      "logits/chosen": -2.5487287044525146,
      "logits/rejected": -2.563288688659668,
      "logps/chosen": -226.3212890625,
      "logps/rejected": -214.44525146484375,
      "loss": 0.746,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6687456965446472,
      "rewards/margins": -0.017180969938635826,
      "rewards/rejected": -0.6515647172927856,
      "step": 571
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.951810670568017e-05,
      "logits/chosen": -2.445035219192505,
      "logits/rejected": -2.3831865787506104,
      "logps/chosen": -214.5970001220703,
      "logps/rejected": -228.41064453125,
      "loss": 0.7696,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.62672358751297,
      "rewards/margins": -0.040074095129966736,
      "rewards/rejected": -0.5866495370864868,
      "step": 572
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.9516069242141885e-05,
      "logits/chosen": -2.321072578430176,
      "logits/rejected": -2.4207139015197754,
      "logps/chosen": -203.04151916503906,
      "logps/rejected": -206.2476806640625,
      "loss": 0.766,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6442697644233704,
      "rewards/margins": -0.051033951342105865,
      "rewards/rejected": -0.5932357907295227,
      "step": 573
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.951402752254219e-05,
      "logits/chosen": -2.3598363399505615,
      "logits/rejected": -2.510725498199463,
      "logps/chosen": -160.54183959960938,
      "logps/rejected": -181.72743225097656,
      "loss": 0.613,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.45199206471443176,
      "rewards/margins": 0.4041283130645752,
      "rewards/rejected": -0.8561204075813293,
      "step": 574
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.951198154723552e-05,
      "logits/chosen": -2.550727605819702,
      "logits/rejected": -2.5166475772857666,
      "logps/chosen": -150.58238220214844,
      "logps/rejected": -157.48434448242188,
      "loss": 0.6906,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3471527397632599,
      "rewards/margins": 0.06844709068536758,
      "rewards/rejected": -0.41559985280036926,
      "step": 575
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.950993131657706e-05,
      "logits/chosen": -2.32822847366333,
      "logits/rejected": -2.3608272075653076,
      "logps/chosen": -183.2533721923828,
      "logps/rejected": -155.5178680419922,
      "loss": 0.8822,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.7413718700408936,
      "rewards/margins": -0.2268693447113037,
      "rewards/rejected": -0.5145025849342346,
      "step": 576
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.9507876830922755e-05,
      "logits/chosen": -2.426633358001709,
      "logits/rejected": -2.350067138671875,
      "logps/chosen": -159.07473754882812,
      "logps/rejected": -177.24722290039062,
      "loss": 0.6643,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4699227511882782,
      "rewards/margins": 0.37284964323043823,
      "rewards/rejected": -0.842772364616394,
      "step": 577
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.950581809062925e-05,
      "logits/chosen": -2.3766255378723145,
      "logits/rejected": -2.411029815673828,
      "logps/chosen": -211.9445343017578,
      "logps/rejected": -224.93450927734375,
      "loss": 0.7549,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5890372395515442,
      "rewards/margins": 0.09359408915042877,
      "rewards/rejected": -0.6826313138008118,
      "step": 578
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.950375509605396e-05,
      "logits/chosen": -2.3283326625823975,
      "logits/rejected": -2.1340157985687256,
      "logps/chosen": -159.19125366210938,
      "logps/rejected": -161.9340362548828,
      "loss": 0.7342,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4637477993965149,
      "rewards/margins": -0.048650771379470825,
      "rewards/rejected": -0.41509705781936646,
      "step": 579
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.9501687847555016e-05,
      "logits/chosen": -2.4342098236083984,
      "logits/rejected": -2.4008848667144775,
      "logps/chosen": -214.58380126953125,
      "logps/rejected": -184.8182373046875,
      "loss": 0.7425,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.45641300082206726,
      "rewards/margins": -0.036872051656246185,
      "rewards/rejected": -0.4195409417152405,
      "step": 580
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.949961634549131e-05,
      "logits/chosen": -2.4010324478149414,
      "logits/rejected": -2.397801160812378,
      "logps/chosen": -175.51223754882812,
      "logps/rejected": -165.90208435058594,
      "loss": 0.6852,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3178666830062866,
      "rewards/margins": 0.06419295072555542,
      "rewards/rejected": -0.38205963373184204,
      "step": 581
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.949754059022246e-05,
      "logits/chosen": -2.494157075881958,
      "logits/rejected": -2.560157537460327,
      "logps/chosen": -172.84835815429688,
      "logps/rejected": -178.97540283203125,
      "loss": 0.6419,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2800545394420624,
      "rewards/margins": 0.1691555678844452,
      "rewards/rejected": -0.44921010732650757,
      "step": 582
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.949546058210881e-05,
      "logits/chosen": -2.209298610687256,
      "logits/rejected": -2.293645143508911,
      "logps/chosen": -194.64381408691406,
      "logps/rejected": -237.14974975585938,
      "loss": 0.5891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.257599413394928,
      "rewards/margins": 0.3168815076351166,
      "rewards/rejected": -0.5744808912277222,
      "step": 583
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.9493376321511464e-05,
      "logits/chosen": -2.446950912475586,
      "logits/rejected": -2.4006083011627197,
      "logps/chosen": -199.42282104492188,
      "logps/rejected": -196.51849365234375,
      "loss": 0.6841,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.35894283652305603,
      "rewards/margins": 0.1623423993587494,
      "rewards/rejected": -0.5212852358818054,
      "step": 584
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.9491287808792265e-05,
      "logits/chosen": -2.3545334339141846,
      "logits/rejected": -2.452188491821289,
      "logps/chosen": -198.55056762695312,
      "logps/rejected": -195.9398956298828,
      "loss": 0.6768,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4273378551006317,
      "rewards/margins": 0.11289644986391068,
      "rewards/rejected": -0.5402343273162842,
      "step": 585
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.948919504431376e-05,
      "logits/chosen": -2.687852144241333,
      "logits/rejected": -2.6648173332214355,
      "logps/chosen": -215.23638916015625,
      "logps/rejected": -221.06588745117188,
      "loss": 0.6981,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.30441591143608093,
      "rewards/margins": 0.08797214180231094,
      "rewards/rejected": -0.39238807559013367,
      "step": 586
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.948709802843929e-05,
      "logits/chosen": -2.2244153022766113,
      "logits/rejected": -2.2272024154663086,
      "logps/chosen": -197.98904418945312,
      "logps/rejected": -188.39773559570312,
      "loss": 0.6626,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.27749666571617126,
      "rewards/margins": 0.12206709384918213,
      "rewards/rejected": -0.3995637595653534,
      "step": 587
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.9484996761532886e-05,
      "logits/chosen": -2.471318006515503,
      "logits/rejected": -2.550818920135498,
      "logps/chosen": -192.51039123535156,
      "logps/rejected": -192.6507568359375,
      "loss": 0.7235,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.27331510186195374,
      "rewards/margins": -0.019602246582508087,
      "rewards/rejected": -0.25371283292770386,
      "step": 588
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.9482891243959335e-05,
      "logits/chosen": -2.4945497512817383,
      "logits/rejected": -2.554481029510498,
      "logps/chosen": -187.52774047851562,
      "logps/rejected": -227.6110076904297,
      "loss": 0.5713,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2509083151817322,
      "rewards/margins": 0.3469720482826233,
      "rewards/rejected": -0.5978803038597107,
      "step": 589
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.948078147608416e-05,
      "logits/chosen": -2.5424046516418457,
      "logits/rejected": -2.58070707321167,
      "logps/chosen": -181.62966918945312,
      "logps/rejected": -219.62698364257812,
      "loss": 0.6805,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4028276205062866,
      "rewards/margins": 0.0835791528224945,
      "rewards/rejected": -0.48640677332878113,
      "step": 590
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.9478667458273624e-05,
      "logits/chosen": -2.558363199234009,
      "logits/rejected": -2.533869743347168,
      "logps/chosen": -207.1812286376953,
      "logps/rejected": -201.0562286376953,
      "loss": 0.6612,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.24153894186019897,
      "rewards/margins": 0.12561962008476257,
      "rewards/rejected": -0.36715856194496155,
      "step": 591
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.9476549190894725e-05,
      "logits/chosen": -2.4393560886383057,
      "logits/rejected": -2.450369358062744,
      "logps/chosen": -153.7821044921875,
      "logps/rejected": -153.88775634765625,
      "loss": 0.686,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.23368775844573975,
      "rewards/margins": 0.0638914704322815,
      "rewards/rejected": -0.29757922887802124,
      "step": 592
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.947442667431522e-05,
      "logits/chosen": -2.6815226078033447,
      "logits/rejected": -2.506643056869507,
      "logps/chosen": -229.4060516357422,
      "logps/rejected": -240.65362548828125,
      "loss": 0.7512,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3724709153175354,
      "rewards/margins": -0.04746837168931961,
      "rewards/rejected": -0.325002521276474,
      "step": 593
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.9472299908903555e-05,
      "logits/chosen": -2.3556230068206787,
      "logits/rejected": -2.4480998516082764,
      "logps/chosen": -128.82077026367188,
      "logps/rejected": -170.01109313964844,
      "loss": 0.6209,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1109066978096962,
      "rewards/margins": 0.2432006597518921,
      "rewards/rejected": -0.3541073799133301,
      "step": 594
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.947016889502895e-05,
      "logits/chosen": -2.5540051460266113,
      "logits/rejected": -2.549333095550537,
      "logps/chosen": -178.5426788330078,
      "logps/rejected": -172.33694458007812,
      "loss": 0.7123,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3693382441997528,
      "rewards/margins": -0.0018721502274274826,
      "rewards/rejected": -0.36746612191200256,
      "step": 595
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.946803363306137e-05,
      "logits/chosen": -2.076066255569458,
      "logits/rejected": -2.011690139770508,
      "logps/chosen": -194.8577880859375,
      "logps/rejected": -176.207763671875,
      "loss": 0.7698,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.26399266719818115,
      "rewards/margins": -0.08310166001319885,
      "rewards/rejected": -0.1808909922838211,
      "step": 596
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.946589412337149e-05,
      "logits/chosen": -2.2557950019836426,
      "logits/rejected": -2.3377816677093506,
      "logps/chosen": -148.5780029296875,
      "logps/rejected": -170.98593139648438,
      "loss": 0.6012,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07843692600727081,
      "rewards/margins": 0.26211532950401306,
      "rewards/rejected": -0.18367838859558105,
      "step": 597
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.9463750366330734e-05,
      "logits/chosen": -2.2877843379974365,
      "logits/rejected": -2.359644889831543,
      "logps/chosen": -187.41412353515625,
      "logps/rejected": -194.01309204101562,
      "loss": 0.734,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3197644352912903,
      "rewards/margins": -0.021138954907655716,
      "rewards/rejected": -0.29862549901008606,
      "step": 598
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.9461602362311275e-05,
      "logits/chosen": -2.493452548980713,
      "logits/rejected": -2.5441408157348633,
      "logps/chosen": -209.37081909179688,
      "logps/rejected": -173.10861206054688,
      "loss": 0.7112,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.324692964553833,
      "rewards/margins": 0.05840418487787247,
      "rewards/rejected": -0.3830971419811249,
      "step": 599
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.9459450111686e-05,
      "logits/chosen": -2.4824540615081787,
      "logits/rejected": -2.477165937423706,
      "logps/chosen": -191.7369384765625,
      "logps/rejected": -181.0056915283203,
      "loss": 0.692,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.38107597827911377,
      "rewards/margins": 0.05237661302089691,
      "rewards/rejected": -0.4334526062011719,
      "step": 600
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.945729361482856e-05,
      "logits/chosen": -2.450261116027832,
      "logits/rejected": -2.500009536743164,
      "logps/chosen": -232.3407440185547,
      "logps/rejected": -212.10638427734375,
      "loss": 0.5732,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.19914206862449646,
      "rewards/margins": 0.3821813762187958,
      "rewards/rejected": -0.5813234448432922,
      "step": 601
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.9455132872113316e-05,
      "logits/chosen": -2.3420450687408447,
      "logits/rejected": -2.3474817276000977,
      "logps/chosen": -216.38914489746094,
      "logps/rejected": -224.3979949951172,
      "loss": 0.6047,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1369827687740326,
      "rewards/margins": 0.2419712096452713,
      "rewards/rejected": -0.3789539933204651,
      "step": 602
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.945296788391539e-05,
      "logits/chosen": -2.3396308422088623,
      "logits/rejected": -2.3148958683013916,
      "logps/chosen": -169.25946044921875,
      "logps/rejected": -197.1276397705078,
      "loss": 0.7964,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.32238638401031494,
      "rewards/margins": -0.1470273733139038,
      "rewards/rejected": -0.17535902559757233,
      "step": 603
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.945079865061062e-05,
      "logits/chosen": -2.275747537612915,
      "logits/rejected": -2.2709038257598877,
      "logps/chosen": -139.6877899169922,
      "logps/rejected": -148.5768585205078,
      "loss": 0.6796,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.08058857917785645,
      "rewards/margins": 0.04861634969711304,
      "rewards/rejected": -0.12920492887496948,
      "step": 604
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.94486251725756e-05,
      "logits/chosen": -2.3698954582214355,
      "logits/rejected": -2.453796625137329,
      "logps/chosen": -229.4521484375,
      "logps/rejected": -210.6240234375,
      "loss": 0.6212,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22786223888397217,
      "rewards/margins": 0.20501334965229034,
      "rewards/rejected": -0.4328756034374237,
      "step": 605
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.944644745018765e-05,
      "logits/chosen": -2.4110870361328125,
      "logits/rejected": -2.423731565475464,
      "logps/chosen": -219.2958221435547,
      "logps/rejected": -202.20706176757812,
      "loss": 0.7773,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.37566569447517395,
      "rewards/margins": -0.0947473868727684,
      "rewards/rejected": -0.28091832995414734,
      "step": 606
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.944426548382482e-05,
      "logits/chosen": -2.4861879348754883,
      "logits/rejected": -2.5459907054901123,
      "logps/chosen": -237.8024444580078,
      "logps/rejected": -260.2302551269531,
      "loss": 0.7621,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.42855891585350037,
      "rewards/margins": -0.048416607081890106,
      "rewards/rejected": -0.38014233112335205,
      "step": 607
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.944207927386591e-05,
      "logits/chosen": -2.478734254837036,
      "logits/rejected": -2.5266644954681396,
      "logps/chosen": -218.4932403564453,
      "logps/rejected": -225.3731689453125,
      "loss": 0.7075,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.23466593027114868,
      "rewards/margins": 0.09529409557580948,
      "rewards/rejected": -0.3299599885940552,
      "step": 608
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.9439888820690475e-05,
      "logits/chosen": -2.5482990741729736,
      "logits/rejected": -2.544922113418579,
      "logps/chosen": -186.0775909423828,
      "logps/rejected": -197.2762908935547,
      "loss": 0.5692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.01272206474095583,
      "rewards/margins": 0.301639199256897,
      "rewards/rejected": -0.3143612742424011,
      "step": 609
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.943769412467875e-05,
      "logits/chosen": -2.2644476890563965,
      "logits/rejected": -2.3761229515075684,
      "logps/chosen": -215.5855712890625,
      "logps/rejected": -246.0447998046875,
      "loss": 0.5599,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19982381165027618,
      "rewards/margins": 0.4122087359428406,
      "rewards/rejected": -0.6120326519012451,
      "step": 610
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.943549518621176e-05,
      "logits/chosen": -2.4406063556671143,
      "logits/rejected": -2.529265880584717,
      "logps/chosen": -175.3257598876953,
      "logps/rejected": -188.4786376953125,
      "loss": 0.6989,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.19547346234321594,
      "rewards/margins": 0.05771362781524658,
      "rewards/rejected": -0.2531871199607849,
      "step": 611
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.9433292005671236e-05,
      "logits/chosen": -2.4887635707855225,
      "logits/rejected": -2.48945951461792,
      "logps/chosen": -187.1964111328125,
      "logps/rejected": -231.7959747314453,
      "loss": 0.6501,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21680843830108643,
      "rewards/margins": 0.15266810357570648,
      "rewards/rejected": -0.3694765567779541,
      "step": 612
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.943108458343968e-05,
      "logits/chosen": -2.4516611099243164,
      "logits/rejected": -2.4493842124938965,
      "logps/chosen": -175.4194793701172,
      "logps/rejected": -185.060546875,
      "loss": 0.6172,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2488647699356079,
      "rewards/margins": 0.22184765338897705,
      "rewards/rejected": -0.47071248292922974,
      "step": 613
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.942887291990028e-05,
      "logits/chosen": -2.636701822280884,
      "logits/rejected": -2.5966298580169678,
      "logps/chosen": -194.91783142089844,
      "logps/rejected": -182.28858947753906,
      "loss": 0.6595,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.04664868861436844,
      "rewards/margins": 0.11729076504707336,
      "rewards/rejected": -0.1639394611120224,
      "step": 614
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.9426657015436994e-05,
      "logits/chosen": -2.303603172302246,
      "logits/rejected": -2.2422983646392822,
      "logps/chosen": -144.08590698242188,
      "logps/rejected": -148.51541137695312,
      "loss": 0.7636,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.18376705050468445,
      "rewards/margins": -0.09818767011165619,
      "rewards/rejected": -0.08557939529418945,
      "step": 615
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.9424436870434515e-05,
      "logits/chosen": -2.7395613193511963,
      "logits/rejected": -2.747368097305298,
      "logps/chosen": -226.32354736328125,
      "logps/rejected": -212.107421875,
      "loss": 0.7233,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.19879278540611267,
      "rewards/margins": 0.0006840275600552559,
      "rewards/rejected": -0.19947680830955505,
      "step": 616
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.942221248527826e-05,
      "logits/chosen": -2.493027448654175,
      "logits/rejected": -2.686164140701294,
      "logps/chosen": -166.07200622558594,
      "logps/rejected": -189.38475036621094,
      "loss": 0.8368,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.28490307927131653,
      "rewards/margins": -0.22959473729133606,
      "rewards/rejected": -0.05530836060643196,
      "step": 617
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.94199838603544e-05,
      "logits/chosen": -2.3346376419067383,
      "logits/rejected": -2.4675793647766113,
      "logps/chosen": -177.6221160888672,
      "logps/rejected": -186.67446899414062,
      "loss": 0.4898,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1148318499326706,
      "rewards/margins": 0.5469637513160706,
      "rewards/rejected": -0.4321318566799164,
      "step": 618
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.941775099604983e-05,
      "logits/chosen": -2.5135788917541504,
      "logits/rejected": -2.6071126461029053,
      "logps/chosen": -186.59161376953125,
      "logps/rejected": -205.1910400390625,
      "loss": 0.6155,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.029235512018203735,
      "rewards/margins": 0.20693793892860413,
      "rewards/rejected": -0.23617346584796906,
      "step": 619
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.941551389275217e-05,
      "logits/chosen": -2.49117374420166,
      "logits/rejected": -2.5229427814483643,
      "logps/chosen": -208.4278106689453,
      "logps/rejected": -191.97398376464844,
      "loss": 0.6074,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18853022158145905,
      "rewards/margins": 0.2418961077928543,
      "rewards/rejected": -0.43042635917663574,
      "step": 620
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.941327255084979e-05,
      "logits/chosen": -2.4580295085906982,
      "logits/rejected": -2.4623186588287354,
      "logps/chosen": -191.52886962890625,
      "logps/rejected": -169.84417724609375,
      "loss": 0.7945,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2514897584915161,
      "rewards/margins": -0.04441142827272415,
      "rewards/rejected": -0.20707830786705017,
      "step": 621
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.9411026970731805e-05,
      "logits/chosen": -2.5821356773376465,
      "logits/rejected": -2.619976758956909,
      "logps/chosen": -203.17149353027344,
      "logps/rejected": -217.1163330078125,
      "loss": 0.6965,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.33491429686546326,
      "rewards/margins": 0.05494851991534233,
      "rewards/rejected": -0.3898628354072571,
      "step": 622
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.9408777152788044e-05,
      "logits/chosen": -2.464843273162842,
      "logits/rejected": -2.5318896770477295,
      "logps/chosen": -169.26376342773438,
      "logps/rejected": -182.11395263671875,
      "loss": 0.6721,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.17076966166496277,
      "rewards/margins": 0.10583178699016571,
      "rewards/rejected": -0.2766014337539673,
      "step": 623
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.9406523097409085e-05,
      "logits/chosen": -2.262134075164795,
      "logits/rejected": -2.329742670059204,
      "logps/chosen": -153.96200561523438,
      "logps/rejected": -157.85238647460938,
      "loss": 0.6468,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.13661958277225494,
      "rewards/margins": 0.20581209659576416,
      "rewards/rejected": -0.3424316644668579,
      "step": 624
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.940426480498623e-05,
      "logits/chosen": -2.403775453567505,
      "logits/rejected": -2.459458351135254,
      "logps/chosen": -167.8171844482422,
      "logps/rejected": -221.13282775878906,
      "loss": 0.6038,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2118593454360962,
      "rewards/margins": 0.26167717576026917,
      "rewards/rejected": -0.47353655099868774,
      "step": 625
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.940200227591154e-05,
      "logits/chosen": -1.9581964015960693,
      "logits/rejected": -2.0107665061950684,
      "logps/chosen": -124.81986236572266,
      "logps/rejected": -138.32220458984375,
      "loss": 0.6216,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.05847623199224472,
      "rewards/margins": 0.22099967300891876,
      "rewards/rejected": -0.2794759273529053,
      "step": 626
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.9399735510577796e-05,
      "logits/chosen": -2.2749149799346924,
      "logits/rejected": -2.283113718032837,
      "logps/chosen": -155.7340087890625,
      "logps/rejected": -150.0223388671875,
      "loss": 0.6522,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1051105409860611,
      "rewards/margins": 0.16768980026245117,
      "rewards/rejected": -0.27280035614967346,
      "step": 627
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.93974645093785e-05,
      "logits/chosen": -2.4072506427764893,
      "logits/rejected": -2.4812541007995605,
      "logps/chosen": -195.17230224609375,
      "logps/rejected": -194.09622192382812,
      "loss": 0.6298,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.06678425520658493,
      "rewards/margins": 0.17194166779518127,
      "rewards/rejected": -0.2387259304523468,
      "step": 628
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.9395189272707916e-05,
      "logits/chosen": -2.428178310394287,
      "logits/rejected": -2.5200040340423584,
      "logps/chosen": -194.95848083496094,
      "logps/rejected": -215.4308319091797,
      "loss": 0.699,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.37522369623184204,
      "rewards/margins": 0.045931555330753326,
      "rewards/rejected": -0.4211552143096924,
      "step": 629
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.939290980096103e-05,
      "logits/chosen": -2.243135690689087,
      "logits/rejected": -2.25071120262146,
      "logps/chosen": -188.89820861816406,
      "logps/rejected": -219.1158447265625,
      "loss": 0.7187,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2669258415699005,
      "rewards/margins": 0.03279988840222359,
      "rewards/rejected": -0.2997257113456726,
      "step": 630
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.9390626094533565e-05,
      "logits/chosen": -2.496492385864258,
      "logits/rejected": -2.6048879623413086,
      "logps/chosen": -169.56320190429688,
      "logps/rejected": -184.27169799804688,
      "loss": 0.6584,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.31219586730003357,
      "rewards/margins": 0.16650965809822083,
      "rewards/rejected": -0.4787055253982544,
      "step": 631
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.9388338153821976e-05,
      "logits/chosen": -2.256747245788574,
      "logits/rejected": -2.2907259464263916,
      "logps/chosen": -163.09808349609375,
      "logps/rejected": -231.26492309570312,
      "loss": 0.5661,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15426135063171387,
      "rewards/margins": 0.48811718821525574,
      "rewards/rejected": -0.6423785090446472,
      "step": 632
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.938604597922346e-05,
      "logits/chosen": -2.5338540077209473,
      "logits/rejected": -2.580803871154785,
      "logps/chosen": -221.49908447265625,
      "logps/rejected": -222.09754943847656,
      "loss": 0.7293,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.393398642539978,
      "rewards/margins": 0.04478640481829643,
      "rewards/rejected": -0.43818506598472595,
      "step": 633
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.9383749571135946e-05,
      "logits/chosen": -2.4923906326293945,
      "logits/rejected": -2.552914619445801,
      "logps/chosen": -188.54759216308594,
      "logps/rejected": -201.86276245117188,
      "loss": 0.6803,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19801469147205353,
      "rewards/margins": 0.08076779544353485,
      "rewards/rejected": -0.2787824869155884,
      "step": 634
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.938144892995809e-05,
      "logits/chosen": -2.6368439197540283,
      "logits/rejected": -2.681831121444702,
      "logps/chosen": -172.92823791503906,
      "logps/rejected": -153.6282501220703,
      "loss": 0.6554,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1948256492614746,
      "rewards/margins": 0.14521902799606323,
      "rewards/rejected": -0.34004467725753784,
      "step": 635
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.93791440560893e-05,
      "logits/chosen": -2.5344183444976807,
      "logits/rejected": -2.5658974647521973,
      "logps/chosen": -198.16390991210938,
      "logps/rejected": -193.16055297851562,
      "loss": 0.6981,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.06259515881538391,
      "rewards/margins": 0.05056838318705559,
      "rewards/rejected": -0.11316356062889099,
      "step": 636
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.93768349499297e-05,
      "logits/chosen": -2.6175780296325684,
      "logits/rejected": -2.561044216156006,
      "logps/chosen": -194.93130493164062,
      "logps/rejected": -171.37437438964844,
      "loss": 0.865,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.44466546177864075,
      "rewards/margins": -0.2321683019399643,
      "rewards/rejected": -0.21249720454216003,
      "step": 637
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.9374521611880166e-05,
      "logits/chosen": -2.552360773086548,
      "logits/rejected": -2.585489273071289,
      "logps/chosen": -157.82420349121094,
      "logps/rejected": -171.7198028564453,
      "loss": 0.6514,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.19169321656227112,
      "rewards/margins": 0.18479710817337036,
      "rewards/rejected": -0.3764903247356415,
      "step": 638
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.9372204042342295e-05,
      "logits/chosen": -2.55067777633667,
      "logits/rejected": -2.6302952766418457,
      "logps/chosen": -186.92031860351562,
      "logps/rejected": -184.224365234375,
      "loss": 0.676,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2839874029159546,
      "rewards/margins": 0.20806796848773956,
      "rewards/rejected": -0.49205532670021057,
      "step": 639
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.936988224171842e-05,
      "logits/chosen": -2.6061015129089355,
      "logits/rejected": -2.609809637069702,
      "logps/chosen": -209.86328125,
      "logps/rejected": -199.8352813720703,
      "loss": 0.6114,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38792091608047485,
      "rewards/margins": 0.29765814542770386,
      "rewards/rejected": -0.6855790615081787,
      "step": 640
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.936755621041163e-05,
      "logits/chosen": -2.600673198699951,
      "logits/rejected": -2.6286747455596924,
      "logps/chosen": -217.86569213867188,
      "logps/rejected": -208.2232208251953,
      "loss": 0.5999,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3269140124320984,
      "rewards/margins": 0.2362305074930191,
      "rewards/rejected": -0.5631444454193115,
      "step": 641
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.93652259488257e-05,
      "logits/chosen": -2.331010341644287,
      "logits/rejected": -2.406320571899414,
      "logps/chosen": -181.08682250976562,
      "logps/rejected": -231.1434326171875,
      "loss": 0.6473,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18573978543281555,
      "rewards/margins": 0.11375062167644501,
      "rewards/rejected": -0.29949042201042175,
      "step": 642
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.9362891457365194e-05,
      "logits/chosen": -2.5639798641204834,
      "logits/rejected": -2.511385917663574,
      "logps/chosen": -167.39529418945312,
      "logps/rejected": -190.17266845703125,
      "loss": 0.6687,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.23445367813110352,
      "rewards/margins": 0.10250186920166016,
      "rewards/rejected": -0.33695557713508606,
      "step": 643
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.9360552736435386e-05,
      "logits/chosen": -2.484229326248169,
      "logits/rejected": -2.5917751789093018,
      "logps/chosen": -160.66372680664062,
      "logps/rejected": -151.573974609375,
      "loss": 0.7071,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3852943181991577,
      "rewards/margins": 0.021035168319940567,
      "rewards/rejected": -0.40632951259613037,
      "step": 644
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.935820978644228e-05,
      "logits/chosen": -2.4788904190063477,
      "logits/rejected": -2.4254837036132812,
      "logps/chosen": -161.3828887939453,
      "logps/rejected": -179.1514129638672,
      "loss": 0.7489,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.27134445309638977,
      "rewards/margins": -0.01518234983086586,
      "rewards/rejected": -0.2561620771884918,
      "step": 645
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.9355862607792614e-05,
      "logits/chosen": -2.3145663738250732,
      "logits/rejected": -2.3181071281433105,
      "logps/chosen": -175.7195281982422,
      "logps/rejected": -180.3649139404297,
      "loss": 0.7226,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2857952117919922,
      "rewards/margins": -0.0009603817015886307,
      "rewards/rejected": -0.2848348617553711,
      "step": 646
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.9353511200893865e-05,
      "logits/chosen": -2.5312018394470215,
      "logits/rejected": -2.5117855072021484,
      "logps/chosen": -147.04232788085938,
      "logps/rejected": -153.8798065185547,
      "loss": 0.7482,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.2138766497373581,
      "rewards/margins": -0.0894002690911293,
      "rewards/rejected": -0.12447639554738998,
      "step": 647
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.935115556615425e-05,
      "logits/chosen": -2.5504438877105713,
      "logits/rejected": -2.5526883602142334,
      "logps/chosen": -198.34768676757812,
      "logps/rejected": -190.35696411132812,
      "loss": 0.6935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.393574059009552,
      "rewards/margins": 0.1438981145620346,
      "rewards/rejected": -0.5374721884727478,
      "step": 648
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.934879570398272e-05,
      "logits/chosen": -2.450056314468384,
      "logits/rejected": -2.440789222717285,
      "logps/chosen": -198.56884765625,
      "logps/rejected": -234.9272918701172,
      "loss": 0.7365,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5248376131057739,
      "rewards/margins": -0.04359120875597,
      "rewards/rejected": -0.4812464118003845,
      "step": 649
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.9346431614788945e-05,
      "logits/chosen": -2.481201648712158,
      "logits/rejected": -2.540107250213623,
      "logps/chosen": -193.42530822753906,
      "logps/rejected": -196.03887939453125,
      "loss": 0.7004,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.30591022968292236,
      "rewards/margins": 0.07436452805995941,
      "rewards/rejected": -0.3802747428417206,
      "step": 650
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.934406329898334e-05,
      "logits/chosen": -2.4828920364379883,
      "logits/rejected": -2.4486544132232666,
      "logps/chosen": -174.25201416015625,
      "logps/rejected": -167.93795776367188,
      "loss": 0.8821,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4259773790836334,
      "rewards/margins": -0.30132219195365906,
      "rewards/rejected": -0.12465521693229675,
      "step": 651
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.934169075697704e-05,
      "logits/chosen": -2.5756962299346924,
      "logits/rejected": -2.53938364982605,
      "logps/chosen": -261.4378356933594,
      "logps/rejected": -217.41473388671875,
      "loss": 0.7193,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.35280755162239075,
      "rewards/margins": 0.009605936706066132,
      "rewards/rejected": -0.36241352558135986,
      "step": 652
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.9339313989181955e-05,
      "logits/chosen": -2.479517936706543,
      "logits/rejected": -2.4295461177825928,
      "logps/chosen": -168.84185791015625,
      "logps/rejected": -171.62991333007812,
      "loss": 0.7111,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4041588306427002,
      "rewards/margins": 0.014433953911066055,
      "rewards/rejected": -0.41859275102615356,
      "step": 653
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.933693299601067e-05,
      "logits/chosen": -2.2451162338256836,
      "logits/rejected": -2.4006645679473877,
      "logps/chosen": -178.17141723632812,
      "logps/rejected": -210.85440063476562,
      "loss": 0.5213,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0021903254091739655,
      "rewards/margins": 0.5191294550895691,
      "rewards/rejected": -0.516939103603363,
      "step": 654
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.933454777787654e-05,
      "logits/chosen": -2.6251702308654785,
      "logits/rejected": -2.6193838119506836,
      "logps/chosen": -261.71185302734375,
      "logps/rejected": -258.5350646972656,
      "loss": 0.6123,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3498530089855194,
      "rewards/margins": 0.23792549967765808,
      "rewards/rejected": -0.5877785086631775,
      "step": 655
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.933215833519366e-05,
      "logits/chosen": -2.515723466873169,
      "logits/rejected": -2.4995017051696777,
      "logps/chosen": -170.08506774902344,
      "logps/rejected": -171.64215087890625,
      "loss": 0.6167,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03253903239965439,
      "rewards/margins": 0.2270340621471405,
      "rewards/rejected": -0.2595731019973755,
      "step": 656
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.9329764668376824e-05,
      "logits/chosen": -2.3417258262634277,
      "logits/rejected": -2.5132431983947754,
      "logps/chosen": -182.23594665527344,
      "logps/rejected": -204.4223175048828,
      "loss": 0.5153,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08154846727848053,
      "rewards/margins": 0.5844264626502991,
      "rewards/rejected": -0.6659749746322632,
      "step": 657
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.932736677784158e-05,
      "logits/chosen": -2.406738758087158,
      "logits/rejected": -2.4287314414978027,
      "logps/chosen": -175.0724639892578,
      "logps/rejected": -212.99569702148438,
      "loss": 0.6727,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5249559879302979,
      "rewards/margins": 0.10958628356456757,
      "rewards/rejected": -0.6345422863960266,
      "step": 658
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.9324964664004226e-05,
      "logits/chosen": -2.4244885444641113,
      "logits/rejected": -2.511654853820801,
      "logps/chosen": -181.31353759765625,
      "logps/rejected": -189.31832885742188,
      "loss": 0.7071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27095070481300354,
      "rewards/margins": -0.0028193406760692596,
      "rewards/rejected": -0.2681313753128052,
      "step": 659
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.9322558327281773e-05,
      "logits/chosen": -2.5121233463287354,
      "logits/rejected": -2.6510791778564453,
      "logps/chosen": -230.39369201660156,
      "logps/rejected": -241.02325439453125,
      "loss": 0.4769,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.02516031824052334,
      "rewards/margins": 0.551730751991272,
      "rewards/rejected": -0.5768911242485046,
      "step": 660
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.9320147768091954e-05,
      "logits/chosen": -2.076624631881714,
      "logits/rejected": -2.117382287979126,
      "logps/chosen": -228.64761352539062,
      "logps/rejected": -211.5120849609375,
      "loss": 0.696,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.37209880352020264,
      "rewards/margins": 0.16731634736061096,
      "rewards/rejected": -0.5394151210784912,
      "step": 661
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.931773298685326e-05,
      "logits/chosen": -2.3631484508514404,
      "logits/rejected": -2.424973249435425,
      "logps/chosen": -174.7392120361328,
      "logps/rejected": -198.92578125,
      "loss": 0.6354,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23427049815654755,
      "rewards/margins": 0.1624229997396469,
      "rewards/rejected": -0.39669349789619446,
      "step": 662
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.93153139839849e-05,
      "logits/chosen": -2.271193742752075,
      "logits/rejected": -2.2948765754699707,
      "logps/chosen": -180.6439971923828,
      "logps/rejected": -192.5905303955078,
      "loss": 0.7982,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.23156273365020752,
      "rewards/margins": -0.15015332400798798,
      "rewards/rejected": -0.08140941709280014,
      "step": 663
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.931289075990682e-05,
      "logits/chosen": -2.6266207695007324,
      "logits/rejected": -2.6377651691436768,
      "logps/chosen": -214.2593536376953,
      "logps/rejected": -202.11019897460938,
      "loss": 0.7092,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.25713613629341125,
      "rewards/margins": 0.02131093665957451,
      "rewards/rejected": -0.2784470319747925,
      "step": 664
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.93104633150397e-05,
      "logits/chosen": -2.558809995651245,
      "logits/rejected": -2.580552816390991,
      "logps/chosen": -200.12879943847656,
      "logps/rejected": -270.4982604980469,
      "loss": 0.5893,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.21998968720436096,
      "rewards/margins": 0.367271363735199,
      "rewards/rejected": -0.5872610807418823,
      "step": 665
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.930803164980496e-05,
      "logits/chosen": -2.324084520339966,
      "logits/rejected": -2.4046638011932373,
      "logps/chosen": -199.2420196533203,
      "logps/rejected": -188.57501220703125,
      "loss": 0.651,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.24593767523765564,
      "rewards/margins": 0.17680732905864716,
      "rewards/rejected": -0.4227449893951416,
      "step": 666
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.930559576462473e-05,
      "logits/chosen": -2.4203665256500244,
      "logits/rejected": -2.5350735187530518,
      "logps/chosen": -203.4146728515625,
      "logps/rejected": -220.17127990722656,
      "loss": 0.6676,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.36869651079177856,
      "rewards/margins": 0.1714363694190979,
      "rewards/rejected": -0.5401328802108765,
      "step": 667
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.930315565992189e-05,
      "logits/chosen": -2.5649983882904053,
      "logits/rejected": -2.587256669998169,
      "logps/chosen": -211.79006958007812,
      "logps/rejected": -197.28616333007812,
      "loss": 0.6443,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.41877490282058716,
      "rewards/margins": 0.1472373902797699,
      "rewards/rejected": -0.5660123229026794,
      "step": 668
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.930071133612005e-05,
      "logits/chosen": -2.4042892456054688,
      "logits/rejected": -2.4121689796447754,
      "logps/chosen": -163.4969940185547,
      "logps/rejected": -214.19903564453125,
      "loss": 0.5355,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12853318452835083,
      "rewards/margins": 0.4976612329483032,
      "rewards/rejected": -0.626194417476654,
      "step": 669
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.929826279364357e-05,
      "logits/chosen": -2.6725361347198486,
      "logits/rejected": -2.6962034702301025,
      "logps/chosen": -206.29966735839844,
      "logps/rejected": -225.02146911621094,
      "loss": 0.7439,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4318912625312805,
      "rewards/margins": -0.017002558335661888,
      "rewards/rejected": -0.4148887097835541,
      "step": 670
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.9295810032917486e-05,
      "logits/chosen": -2.6312649250030518,
      "logits/rejected": -2.677178144454956,
      "logps/chosen": -233.12554931640625,
      "logps/rejected": -249.84649658203125,
      "loss": 0.6412,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3292105495929718,
      "rewards/margins": 0.18580739200115204,
      "rewards/rejected": -0.5150179266929626,
      "step": 671
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.929335305436764e-05,
      "logits/chosen": -2.350733995437622,
      "logits/rejected": -2.4738504886627197,
      "logps/chosen": -159.27565002441406,
      "logps/rejected": -197.69430541992188,
      "loss": 0.5996,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3497101664543152,
      "rewards/margins": 0.35428282618522644,
      "rewards/rejected": -0.7039930820465088,
      "step": 672
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.9290891858420554e-05,
      "logits/chosen": -2.341266632080078,
      "logits/rejected": -2.412942409515381,
      "logps/chosen": -221.41571044921875,
      "logps/rejected": -226.8486328125,
      "loss": 0.5985,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.34598851203918457,
      "rewards/margins": 0.2551480233669281,
      "rewards/rejected": -0.6011365652084351,
      "step": 673
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.92884264455035e-05,
      "logits/chosen": -2.302943706512451,
      "logits/rejected": -2.3145551681518555,
      "logps/chosen": -149.34109497070312,
      "logps/rejected": -138.53506469726562,
      "loss": 0.644,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1495213508605957,
      "rewards/margins": 0.15739065408706665,
      "rewards/rejected": -0.30691200494766235,
      "step": 674
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.9285956816044486e-05,
      "logits/chosen": -2.228523015975952,
      "logits/rejected": -2.173222064971924,
      "logps/chosen": -177.70559692382812,
      "logps/rejected": -210.69927978515625,
      "loss": 0.7657,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3950153589248657,
      "rewards/margins": -0.07809535413980484,
      "rewards/rejected": -0.3169199824333191,
      "step": 675
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.9283482970472245e-05,
      "logits/chosen": -2.3849382400512695,
      "logits/rejected": -2.513273000717163,
      "logps/chosen": -185.79124450683594,
      "logps/rejected": -201.27066040039062,
      "loss": 0.6207,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13350968062877655,
      "rewards/margins": 0.2174827605485916,
      "rewards/rejected": -0.35099244117736816,
      "step": 676
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.928100490921625e-05,
      "logits/chosen": -2.458615779876709,
      "logits/rejected": -2.4962515830993652,
      "logps/chosen": -204.7447509765625,
      "logps/rejected": -199.8260955810547,
      "loss": 0.7519,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6041467189788818,
      "rewards/margins": 0.038420408964157104,
      "rewards/rejected": -0.6425671577453613,
      "step": 677
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.927852263270667e-05,
      "logits/chosen": -2.5404415130615234,
      "logits/rejected": -2.5182065963745117,
      "logps/chosen": -208.4690399169922,
      "logps/rejected": -227.29690551757812,
      "loss": 0.6729,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.28778883814811707,
      "rewards/margins": 0.08314257860183716,
      "rewards/rejected": -0.3709314167499542,
      "step": 678
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.9276036141374473e-05,
      "logits/chosen": -2.528585910797119,
      "logits/rejected": -2.5861098766326904,
      "logps/chosen": -211.7762908935547,
      "logps/rejected": -236.7048797607422,
      "loss": 0.7562,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4092783033847809,
      "rewards/margins": -0.03306754678487778,
      "rewards/rejected": -0.3762107491493225,
      "step": 679
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.92735454356513e-05,
      "logits/chosen": -2.172372341156006,
      "logits/rejected": -2.1387619972229004,
      "logps/chosen": -230.44168090820312,
      "logps/rejected": -221.9505615234375,
      "loss": 0.6126,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0995088517665863,
      "rewards/margins": 0.24330440163612366,
      "rewards/rejected": -0.34281325340270996,
      "step": 680
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.927105051596956e-05,
      "logits/chosen": -2.5018787384033203,
      "logits/rejected": -2.6133289337158203,
      "logps/chosen": -159.59141540527344,
      "logps/rejected": -165.89492797851562,
      "loss": 0.6031,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0614471361041069,
      "rewards/margins": 0.24630111455917358,
      "rewards/rejected": -0.3077481985092163,
      "step": 681
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.926855138276236e-05,
      "logits/chosen": -2.532803773880005,
      "logits/rejected": -2.6221201419830322,
      "logps/chosen": -192.40725708007812,
      "logps/rejected": -176.5653839111328,
      "loss": 0.7419,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4705914258956909,
      "rewards/margins": 0.014578044414520264,
      "rewards/rejected": -0.48516952991485596,
      "step": 682
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.9266048036463566e-05,
      "logits/chosen": -2.5754356384277344,
      "logits/rejected": -2.668449878692627,
      "logps/chosen": -223.462646484375,
      "logps/rejected": -249.08425903320312,
      "loss": 0.5664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4348244071006775,
      "rewards/margins": 0.38006237149238586,
      "rewards/rejected": -0.8148868680000305,
      "step": 683
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.926354047750777e-05,
      "logits/chosen": -2.148771047592163,
      "logits/rejected": -2.174349546432495,
      "logps/chosen": -191.0968780517578,
      "logps/rejected": -201.15272521972656,
      "loss": 0.7219,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.25711679458618164,
      "rewards/margins": 0.0128050297498703,
      "rewards/rejected": -0.26992183923721313,
      "step": 684
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.926102870633029e-05,
      "logits/chosen": -2.5497798919677734,
      "logits/rejected": -2.5441908836364746,
      "logps/chosen": -236.3572998046875,
      "logps/rejected": -223.45065307617188,
      "loss": 0.734,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.43852365016937256,
      "rewards/margins": 0.028032146394252777,
      "rewards/rejected": -0.46655580401420593,
      "step": 685
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.925851272336717e-05,
      "logits/chosen": -2.49284029006958,
      "logits/rejected": -2.4962539672851562,
      "logps/chosen": -187.33041381835938,
      "logps/rejected": -161.27676391601562,
      "loss": 0.7157,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4357530474662781,
      "rewards/margins": 0.01825621724128723,
      "rewards/rejected": -0.4540092647075653,
      "step": 686
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.9255992529055195e-05,
      "logits/chosen": -2.4232540130615234,
      "logits/rejected": -2.4808590412139893,
      "logps/chosen": -177.44873046875,
      "logps/rejected": -173.36376953125,
      "loss": 0.5234,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.000440012663602829,
      "rewards/margins": 0.42676159739494324,
      "rewards/rejected": -0.4272015690803528,
      "step": 687
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.925346812383188e-05,
      "logits/chosen": -2.5829415321350098,
      "logits/rejected": -2.539314031600952,
      "logps/chosen": -201.11561584472656,
      "logps/rejected": -157.01829528808594,
      "loss": 0.659,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.18970540165901184,
      "rewards/margins": 0.1848244071006775,
      "rewards/rejected": -0.37452977895736694,
      "step": 688
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.925093950813547e-05,
      "logits/chosen": -2.1912455558776855,
      "logits/rejected": -2.235294818878174,
      "logps/chosen": -158.46246337890625,
      "logps/rejected": -182.1004180908203,
      "loss": 0.7104,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2667257487773895,
      "rewards/margins": 0.0735367089509964,
      "rewards/rejected": -0.3402624726295471,
      "step": 689
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.924840668240495e-05,
      "logits/chosen": -2.2375993728637695,
      "logits/rejected": -2.270219564437866,
      "logps/chosen": -206.09689331054688,
      "logps/rejected": -185.82130432128906,
      "loss": 0.8109,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5142911672592163,
      "rewards/margins": -0.055089227855205536,
      "rewards/rejected": -0.4592019319534302,
      "step": 690
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.924586964708e-05,
      "logits/chosen": -2.683393716812134,
      "logits/rejected": -2.745443344116211,
      "logps/chosen": -211.4540252685547,
      "logps/rejected": -204.97247314453125,
      "loss": 0.6848,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.388435035943985,
      "rewards/margins": 0.09198732674121857,
      "rewards/rejected": -0.48042237758636475,
      "step": 691
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.9243328402601075e-05,
      "logits/chosen": -2.4001381397247314,
      "logits/rejected": -2.446291208267212,
      "logps/chosen": -173.99537658691406,
      "logps/rejected": -170.11166381835938,
      "loss": 0.6884,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3871830105781555,
      "rewards/margins": 0.05776102468371391,
      "rewards/rejected": -0.44494396448135376,
      "step": 692
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.924078294940935e-05,
      "logits/chosen": -2.3540093898773193,
      "logits/rejected": -2.4053566455841064,
      "logps/chosen": -238.71336364746094,
      "logps/rejected": -232.11026000976562,
      "loss": 0.6272,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.38184699416160583,
      "rewards/margins": 0.2840040624141693,
      "rewards/rejected": -0.6658511161804199,
      "step": 693
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.92382332879467e-05,
      "logits/chosen": -2.5318589210510254,
      "logits/rejected": -2.570647954940796,
      "logps/chosen": -182.59814453125,
      "logps/rejected": -190.98745727539062,
      "loss": 0.6351,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4041218161582947,
      "rewards/margins": 0.2801179587841034,
      "rewards/rejected": -0.6842397451400757,
      "step": 694
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.923567941865577e-05,
      "logits/chosen": -2.440756320953369,
      "logits/rejected": -2.4445791244506836,
      "logps/chosen": -174.62588500976562,
      "logps/rejected": -186.00222778320312,
      "loss": 0.6708,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3223167061805725,
      "rewards/margins": 0.1542733609676361,
      "rewards/rejected": -0.4765900671482086,
      "step": 695
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.923312134197991e-05,
      "logits/chosen": -2.3524816036224365,
      "logits/rejected": -2.2868611812591553,
      "logps/chosen": -174.77328491210938,
      "logps/rejected": -193.611083984375,
      "loss": 0.8216,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4883018434047699,
      "rewards/margins": -0.1851535439491272,
      "rewards/rejected": -0.3031483292579651,
      "step": 696
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.9230559058363223e-05,
      "logits/chosen": -2.2453904151916504,
      "logits/rejected": -2.2618868350982666,
      "logps/chosen": -204.50955200195312,
      "logps/rejected": -213.33969116210938,
      "loss": 0.6688,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2155238389968872,
      "rewards/margins": 0.14622408151626587,
      "rewards/rejected": -0.3617479205131531,
      "step": 697
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.922799256825051e-05,
      "logits/chosen": -2.2087559700012207,
      "logits/rejected": -2.289468765258789,
      "logps/chosen": -223.2991943359375,
      "logps/rejected": -193.14785766601562,
      "loss": 0.8451,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.45806586742401123,
      "rewards/margins": -0.23903487622737885,
      "rewards/rejected": -0.21903102099895477,
      "step": 698
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.9225421872087344e-05,
      "logits/chosen": -2.6118435859680176,
      "logits/rejected": -2.5954737663269043,
      "logps/chosen": -218.72201538085938,
      "logps/rejected": -223.23770141601562,
      "loss": 0.6797,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.33901646733283997,
      "rewards/margins": 0.09220410138368607,
      "rewards/rejected": -0.43122053146362305,
      "step": 699
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.922284697031999e-05,
      "logits/chosen": -2.532050371170044,
      "logits/rejected": -2.5437393188476562,
      "logps/chosen": -205.47463989257812,
      "logps/rejected": -194.44210815429688,
      "loss": 0.6153,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.293450266122818,
      "rewards/margins": 0.32601144909858704,
      "rewards/rejected": -0.619461715221405,
      "step": 700
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.922026786339545e-05,
      "logits/chosen": -2.4742062091827393,
      "logits/rejected": -2.507223606109619,
      "logps/chosen": -249.90118408203125,
      "logps/rejected": -243.16616821289062,
      "loss": 0.6483,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3557482063770294,
      "rewards/margins": 0.16462744772434235,
      "rewards/rejected": -0.5203756093978882,
      "step": 701
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.921768455176149e-05,
      "logits/chosen": -2.313598155975342,
      "logits/rejected": -2.38657546043396,
      "logps/chosen": -218.81178283691406,
      "logps/rejected": -273.9117126464844,
      "loss": 0.5901,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16350752115249634,
      "rewards/margins": 0.2863968014717102,
      "rewards/rejected": -0.44990429282188416,
      "step": 702
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.921509703586657e-05,
      "logits/chosen": -2.4725236892700195,
      "logits/rejected": -2.46490216255188,
      "logps/chosen": -204.388916015625,
      "logps/rejected": -202.4940948486328,
      "loss": 0.6274,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11353873461484909,
      "rewards/margins": 0.18249502778053284,
      "rewards/rejected": -0.2960337698459625,
      "step": 703
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.9212505316159876e-05,
      "logits/chosen": -2.4417524337768555,
      "logits/rejected": -2.4327855110168457,
      "logps/chosen": -185.76559448242188,
      "logps/rejected": -185.82675170898438,
      "loss": 0.6754,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16719910502433777,
      "rewards/margins": 0.131285160779953,
      "rewards/rejected": -0.29848429560661316,
      "step": 704
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.920990939309135e-05,
      "logits/chosen": -2.405703544616699,
      "logits/rejected": -2.4366328716278076,
      "logps/chosen": -182.18052673339844,
      "logps/rejected": -177.89529418945312,
      "loss": 0.5939,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0767878070473671,
      "rewards/margins": 0.26920846104621887,
      "rewards/rejected": -0.34599626064300537,
      "step": 705
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.9207309267111654e-05,
      "logits/chosen": -2.267641067504883,
      "logits/rejected": -2.359164237976074,
      "logps/chosen": -167.9193115234375,
      "logps/rejected": -198.30853271484375,
      "loss": 0.5458,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.17353487014770508,
      "rewards/margins": 0.4786756932735443,
      "rewards/rejected": -0.6522104144096375,
      "step": 706
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.920470493867217e-05,
      "logits/chosen": -2.3301897048950195,
      "logits/rejected": -2.287158489227295,
      "logps/chosen": -179.1868133544922,
      "logps/rejected": -173.25205993652344,
      "loss": 0.8096,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3885132968425751,
      "rewards/margins": -0.15448781847953796,
      "rewards/rejected": -0.2340254783630371,
      "step": 707
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.920209640822503e-05,
      "logits/chosen": -2.3286001682281494,
      "logits/rejected": -2.372384786605835,
      "logps/chosen": -183.89163208007812,
      "logps/rejected": -171.33837890625,
      "loss": 0.6243,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.200511634349823,
      "rewards/margins": 0.1888035386800766,
      "rewards/rejected": -0.3893151581287384,
      "step": 708
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.919948367622307e-05,
      "logits/chosen": -2.049145221710205,
      "logits/rejected": -2.1091737747192383,
      "logps/chosen": -165.6485595703125,
      "logps/rejected": -196.3398895263672,
      "loss": 0.6972,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21507728099822998,
      "rewards/margins": 0.1645798683166504,
      "rewards/rejected": -0.37965714931488037,
      "step": 709
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.919686674311987e-05,
      "logits/chosen": -2.2241785526275635,
      "logits/rejected": -2.1966073513031006,
      "logps/chosen": -197.89120483398438,
      "logps/rejected": -222.09132385253906,
      "loss": 0.6355,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12498195469379425,
      "rewards/margins": 0.2979465425014496,
      "rewards/rejected": -0.42292848229408264,
      "step": 710
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.9194245609369746e-05,
      "logits/chosen": -2.3444929122924805,
      "logits/rejected": -2.3332207202911377,
      "logps/chosen": -240.3154296875,
      "logps/rejected": -229.0690460205078,
      "loss": 0.6236,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.17825375497341156,
      "rewards/margins": 0.23597893118858337,
      "rewards/rejected": -0.41423267126083374,
      "step": 711
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.9191620275427725e-05,
      "logits/chosen": -2.472026824951172,
      "logits/rejected": -2.424983501434326,
      "logps/chosen": -212.58364868164062,
      "logps/rejected": -186.08990478515625,
      "loss": 0.684,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.1622922122478485,
      "rewards/margins": 0.10522370040416718,
      "rewards/rejected": -0.2675159275531769,
      "step": 712
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.9188990741749576e-05,
      "logits/chosen": -2.481387138366699,
      "logits/rejected": -2.4912960529327393,
      "logps/chosen": -259.65753173828125,
      "logps/rejected": -245.88864135742188,
      "loss": 0.6385,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3589114546775818,
      "rewards/margins": 0.13590216636657715,
      "rewards/rejected": -0.49481362104415894,
      "step": 713
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.918635700879179e-05,
      "logits/chosen": -2.4197511672973633,
      "logits/rejected": -2.442139148712158,
      "logps/chosen": -180.3101806640625,
      "logps/rejected": -201.78469848632812,
      "loss": 0.696,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.27619680762290955,
      "rewards/margins": 0.04660003259778023,
      "rewards/rejected": -0.32279688119888306,
      "step": 714
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.918371907701159e-05,
      "logits/chosen": -2.333615303039551,
      "logits/rejected": -2.3523952960968018,
      "logps/chosen": -176.4866485595703,
      "logps/rejected": -190.32089233398438,
      "loss": 0.755,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.353998064994812,
      "rewards/margins": -0.033618077635765076,
      "rewards/rejected": -0.32037997245788574,
      "step": 715
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.9181076946866944e-05,
      "logits/chosen": -2.45198655128479,
      "logits/rejected": -2.432290554046631,
      "logps/chosen": -178.83279418945312,
      "logps/rejected": -188.7766876220703,
      "loss": 0.7776,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4141733944416046,
      "rewards/margins": -0.06833554804325104,
      "rewards/rejected": -0.3458378314971924,
      "step": 716
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.917843061881652e-05,
      "logits/chosen": -2.4485764503479004,
      "logits/rejected": -2.412081241607666,
      "logps/chosen": -229.4910430908203,
      "logps/rejected": -225.20999145507812,
      "loss": 0.7713,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4986688494682312,
      "rewards/margins": -0.1049906462430954,
      "rewards/rejected": -0.3936781883239746,
      "step": 717
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.9175780093319723e-05,
      "logits/chosen": -2.619525194168091,
      "logits/rejected": -2.5976409912109375,
      "logps/chosen": -152.8652801513672,
      "logps/rejected": -190.03179931640625,
      "loss": 0.6384,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.044289495795965195,
      "rewards/margins": 0.1848900467157364,
      "rewards/rejected": -0.2291795164346695,
      "step": 718
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.91731253708367e-05,
      "logits/chosen": -2.2807188034057617,
      "logits/rejected": -2.338017463684082,
      "logps/chosen": -182.47259521484375,
      "logps/rejected": -196.39678955078125,
      "loss": 0.6927,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.26287615299224854,
      "rewards/margins": 0.050617627799510956,
      "rewards/rejected": -0.3134937882423401,
      "step": 719
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.9170466451828326e-05,
      "logits/chosen": -2.2561631202697754,
      "logits/rejected": -2.3567490577697754,
      "logps/chosen": -242.33407592773438,
      "logps/rejected": -263.4303283691406,
      "loss": 0.7338,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2756967544555664,
      "rewards/margins": 0.13067740201950073,
      "rewards/rejected": -0.40637409687042236,
      "step": 720
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.916780333675618e-05,
      "logits/chosen": -2.5038459300994873,
      "logits/rejected": -2.500086545944214,
      "logps/chosen": -187.3552703857422,
      "logps/rejected": -187.51268005371094,
      "loss": 0.6628,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2900952398777008,
      "rewards/margins": 0.16025833785533905,
      "rewards/rejected": -0.4503535330295563,
      "step": 721
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.9165136026082604e-05,
      "logits/chosen": -2.2002265453338623,
      "logits/rejected": -2.226296901702881,
      "logps/chosen": -200.2354278564453,
      "logps/rejected": -195.80328369140625,
      "loss": 0.6676,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4443664848804474,
      "rewards/margins": 0.14572381973266602,
      "rewards/rejected": -0.5900903344154358,
      "step": 722
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.916246452027063e-05,
      "logits/chosen": -2.1249847412109375,
      "logits/rejected": -2.116081953048706,
      "logps/chosen": -214.81161499023438,
      "logps/rejected": -220.05429077148438,
      "loss": 0.664,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.25689569115638733,
      "rewards/margins": 0.1331162005662918,
      "rewards/rejected": -0.39001190662384033,
      "step": 723
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.9159788819784064e-05,
      "logits/chosen": -2.3239493370056152,
      "logits/rejected": -2.3413610458374023,
      "logps/chosen": -209.00692749023438,
      "logps/rejected": -198.1204376220703,
      "loss": 0.627,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2650049328804016,
      "rewards/margins": 0.20938965678215027,
      "rewards/rejected": -0.47439464926719666,
      "step": 724
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.9157108925087405e-05,
      "logits/chosen": -2.420534610748291,
      "logits/rejected": -2.442646026611328,
      "logps/chosen": -211.38424682617188,
      "logps/rejected": -207.95278930664062,
      "loss": 0.5418,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.22766509652137756,
      "rewards/margins": 0.3782108724117279,
      "rewards/rejected": -0.6058759689331055,
      "step": 725
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.915442483664588e-05,
      "logits/chosen": -2.6422438621520996,
      "logits/rejected": -2.4694361686706543,
      "logps/chosen": -173.2271728515625,
      "logps/rejected": -145.02798461914062,
      "loss": 0.8003,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.25257498025894165,
      "rewards/margins": -0.11290948837995529,
      "rewards/rejected": -0.13966546952724457,
      "step": 726
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.915173655492547e-05,
      "logits/chosen": -2.5392398834228516,
      "logits/rejected": -2.4417877197265625,
      "logps/chosen": -228.2000732421875,
      "logps/rejected": -199.39288330078125,
      "loss": 0.609,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.23776192963123322,
      "rewards/margins": 0.23510348796844482,
      "rewards/rejected": -0.47286540269851685,
      "step": 727
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.914904408039286e-05,
      "logits/chosen": -2.4822921752929688,
      "logits/rejected": -2.5780792236328125,
      "logps/chosen": -194.31045532226562,
      "logps/rejected": -235.24607849121094,
      "loss": 0.5732,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3551061451435089,
      "rewards/margins": 0.39131835103034973,
      "rewards/rejected": -0.7464244961738586,
      "step": 728
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.914634741351547e-05,
      "logits/chosen": -2.4005560874938965,
      "logits/rejected": -2.4636473655700684,
      "logps/chosen": -222.8491668701172,
      "logps/rejected": -229.69638061523438,
      "loss": 0.7059,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.444613516330719,
      "rewards/margins": 0.05394814535975456,
      "rewards/rejected": -0.49856165051460266,
      "step": 729
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.914364655476146e-05,
      "logits/chosen": -2.541283369064331,
      "logits/rejected": -2.4821250438690186,
      "logps/chosen": -199.74075317382812,
      "logps/rejected": -244.3222198486328,
      "loss": 0.5754,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24406713247299194,
      "rewards/margins": 0.44718241691589355,
      "rewards/rejected": -0.6912496089935303,
      "step": 730
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.914094150459969e-05,
      "logits/chosen": -2.3721487522125244,
      "logits/rejected": -2.4787163734436035,
      "logps/chosen": -185.42625427246094,
      "logps/rejected": -214.8418426513672,
      "loss": 0.741,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5116318464279175,
      "rewards/margins": -0.03801153600215912,
      "rewards/rejected": -0.47362035512924194,
      "step": 731
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.9138232263499784e-05,
      "logits/chosen": -2.5538036823272705,
      "logits/rejected": -2.467003583908081,
      "logps/chosen": -197.03834533691406,
      "logps/rejected": -214.0052947998047,
      "loss": 0.812,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5602855682373047,
      "rewards/margins": -0.166466623544693,
      "rewards/rejected": -0.3938189446926117,
      "step": 732
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.913551883193206e-05,
      "logits/chosen": -2.4558639526367188,
      "logits/rejected": -2.5085341930389404,
      "logps/chosen": -225.220947265625,
      "logps/rejected": -227.97213745117188,
      "loss": 0.6471,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2800041437149048,
      "rewards/margins": 0.34519922733306885,
      "rewards/rejected": -0.6252033114433289,
      "step": 733
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.9132801210367586e-05,
      "logits/chosen": -2.405069351196289,
      "logits/rejected": -2.4901866912841797,
      "logps/chosen": -152.56527709960938,
      "logps/rejected": -152.64671325683594,
      "loss": 0.6334,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3601643145084381,
      "rewards/margins": 0.2201770395040512,
      "rewards/rejected": -0.5803413987159729,
      "step": 734
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.913007939927814e-05,
      "logits/chosen": -2.3362531661987305,
      "logits/rejected": -2.3917181491851807,
      "logps/chosen": -193.826904296875,
      "logps/rejected": -231.73898315429688,
      "loss": 0.6171,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5505282282829285,
      "rewards/margins": 0.3396502137184143,
      "rewards/rejected": -0.890178382396698,
      "step": 735
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.912735339913625e-05,
      "logits/chosen": -2.132725715637207,
      "logits/rejected": -2.2681026458740234,
      "logps/chosen": -198.4554443359375,
      "logps/rejected": -205.1267852783203,
      "loss": 0.6419,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.34412312507629395,
      "rewards/margins": 0.1572694331407547,
      "rewards/rejected": -0.5013926029205322,
      "step": 736
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.912462321041513e-05,
      "logits/chosen": -2.352013349533081,
      "logits/rejected": -2.474015235900879,
      "logps/chosen": -216.76731872558594,
      "logps/rejected": -208.0941162109375,
      "loss": 0.7258,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.49095839262008667,
      "rewards/margins": -0.004371330142021179,
      "rewards/rejected": -0.4865871071815491,
      "step": 737
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.9121888833588795e-05,
      "logits/chosen": -2.395151138305664,
      "logits/rejected": -2.40535306930542,
      "logps/chosen": -163.3991241455078,
      "logps/rejected": -184.1988525390625,
      "loss": 0.6083,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.35962533950805664,
      "rewards/margins": 0.20414502918720245,
      "rewards/rejected": -0.5637703537940979,
      "step": 738
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.91191502691319e-05,
      "logits/chosen": -2.3076629638671875,
      "logits/rejected": -2.460707426071167,
      "logps/chosen": -163.59388732910156,
      "logps/rejected": -190.1605987548828,
      "loss": 0.6092,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2011553794145584,
      "rewards/margins": 0.24963317811489105,
      "rewards/rejected": -0.45078855752944946,
      "step": 739
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.911640751751988e-05,
      "logits/chosen": -2.419591188430786,
      "logits/rejected": -2.537069797515869,
      "logps/chosen": -250.05996704101562,
      "logps/rejected": -225.0284881591797,
      "loss": 0.7242,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6081552505493164,
      "rewards/margins": 0.06761530041694641,
      "rewards/rejected": -0.6757705807685852,
      "step": 740
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.9113660579228886e-05,
      "logits/chosen": -2.4080655574798584,
      "logits/rejected": -2.4866342544555664,
      "logps/chosen": -168.5004119873047,
      "logps/rejected": -216.32675170898438,
      "loss": 0.6338,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5959919691085815,
      "rewards/margins": 0.22804662585258484,
      "rewards/rejected": -0.8240386247634888,
      "step": 741
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.91109094547358e-05,
      "logits/chosen": -2.2880287170410156,
      "logits/rejected": -2.4074809551239014,
      "logps/chosen": -184.07696533203125,
      "logps/rejected": -215.08717346191406,
      "loss": 0.6027,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.49041178822517395,
      "rewards/margins": 0.23776140809059143,
      "rewards/rejected": -0.7281732559204102,
      "step": 742
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.910815414451821e-05,
      "logits/chosen": -2.4181840419769287,
      "logits/rejected": -2.4892990589141846,
      "logps/chosen": -215.46221923828125,
      "logps/rejected": -229.75799560546875,
      "loss": 0.608,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7831647992134094,
      "rewards/margins": 0.38174957036972046,
      "rewards/rejected": -1.1649143695831299,
      "step": 743
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.910539464905447e-05,
      "logits/chosen": -2.2858383655548096,
      "logits/rejected": -2.3867931365966797,
      "logps/chosen": -257.6205749511719,
      "logps/rejected": -225.37481689453125,
      "loss": 0.6948,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5961012840270996,
      "rewards/margins": 0.15920449793338776,
      "rewards/rejected": -0.7553057670593262,
      "step": 744
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.910263096882362e-05,
      "logits/chosen": -2.2314653396606445,
      "logits/rejected": -2.242586851119995,
      "logps/chosen": -157.20448303222656,
      "logps/rejected": -189.92567443847656,
      "loss": 0.7938,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5152949690818787,
      "rewards/margins": -0.07010670006275177,
      "rewards/rejected": -0.4451882541179657,
      "step": 745
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.9099863104305446e-05,
      "logits/chosen": -2.34870982170105,
      "logits/rejected": -2.3678641319274902,
      "logps/chosen": -157.9035186767578,
      "logps/rejected": -194.49575805664062,
      "loss": 0.6431,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.39278292655944824,
      "rewards/margins": 0.1728055626153946,
      "rewards/rejected": -0.5655884742736816,
      "step": 746
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.909709105598045e-05,
      "logits/chosen": -2.290536403656006,
      "logits/rejected": -2.316704034805298,
      "logps/chosen": -173.55892944335938,
      "logps/rejected": -190.94125366210938,
      "loss": 0.6078,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3803100883960724,
      "rewards/margins": 0.2535950839519501,
      "rewards/rejected": -0.6339051723480225,
      "step": 747
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.9094314824329876e-05,
      "logits/chosen": -2.5316176414489746,
      "logits/rejected": -2.4976158142089844,
      "logps/chosen": -206.0629119873047,
      "logps/rejected": -219.75418090820312,
      "loss": 0.8139,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5901838541030884,
      "rewards/margins": -0.1481025069952011,
      "rewards/rejected": -0.4420813322067261,
      "step": 748
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.9091534409835694e-05,
      "logits/chosen": -2.500584363937378,
      "logits/rejected": -2.5805132389068604,
      "logps/chosen": -174.99212646484375,
      "logps/rejected": -207.73011779785156,
      "loss": 0.574,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4852156937122345,
      "rewards/margins": 0.3404516577720642,
      "rewards/rejected": -0.8256674408912659,
      "step": 749
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.908874981298057e-05,
      "logits/chosen": -2.5304508209228516,
      "logits/rejected": -2.6482512950897217,
      "logps/chosen": -160.60971069335938,
      "logps/rejected": -178.6311492919922,
      "loss": 0.7716,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4607134461402893,
      "rewards/margins": -0.038665421307086945,
      "rewards/rejected": -0.42204800248146057,
      "step": 750
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.908596103424794e-05,
      "logits/chosen": -2.247166633605957,
      "logits/rejected": -2.2175076007843018,
      "logps/chosen": -185.73583984375,
      "logps/rejected": -191.45596313476562,
      "loss": 0.8569,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.6400306820869446,
      "rewards/margins": -0.2516973912715912,
      "rewards/rejected": -0.3883333206176758,
      "step": 751
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.908316807412194e-05,
      "logits/chosen": -2.492708206176758,
      "logits/rejected": -2.585394859313965,
      "logps/chosen": -213.05731201171875,
      "logps/rejected": -260.92462158203125,
      "loss": 0.6273,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5991235971450806,
      "rewards/margins": 0.2069501429796219,
      "rewards/rejected": -0.806073784828186,
      "step": 752
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.908037093308742e-05,
      "logits/chosen": -2.5198185443878174,
      "logits/rejected": -2.5527148246765137,
      "logps/chosen": -238.94021606445312,
      "logps/rejected": -262.2608337402344,
      "loss": 0.699,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.38716763257980347,
      "rewards/margins": 0.033374398946762085,
      "rewards/rejected": -0.42054203152656555,
      "step": 753
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.907756961162999e-05,
      "logits/chosen": -2.26259708404541,
      "logits/rejected": -2.2545177936553955,
      "logps/chosen": -150.26882934570312,
      "logps/rejected": -227.8256072998047,
      "loss": 0.579,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3753708302974701,
      "rewards/margins": 0.4033656418323517,
      "rewards/rejected": -0.7787365317344666,
      "step": 754
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.907476411023596e-05,
      "logits/chosen": -2.3281123638153076,
      "logits/rejected": -2.330186128616333,
      "logps/chosen": -149.7628936767578,
      "logps/rejected": -193.10093688964844,
      "loss": 0.7339,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6727653741836548,
      "rewards/margins": 0.06607560813426971,
      "rewards/rejected": -0.7388409376144409,
      "step": 755
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.907195442939237e-05,
      "logits/chosen": -2.4469387531280518,
      "logits/rejected": -2.509931802749634,
      "logps/chosen": -174.05670166015625,
      "logps/rejected": -189.08465576171875,
      "loss": 0.701,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5176682472229004,
      "rewards/margins": 0.04994421452283859,
      "rewards/rejected": -0.5676124095916748,
      "step": 756
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.906914056958699e-05,
      "logits/chosen": -2.6258902549743652,
      "logits/rejected": -2.6027467250823975,
      "logps/chosen": -208.80203247070312,
      "logps/rejected": -200.4151611328125,
      "loss": 0.7588,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6569511294364929,
      "rewards/margins": -0.044889748096466064,
      "rewards/rejected": -0.6120614409446716,
      "step": 757
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.906632253130833e-05,
      "logits/chosen": -2.5387301445007324,
      "logits/rejected": -2.496183156967163,
      "logps/chosen": -193.3511199951172,
      "logps/rejected": -199.80130004882812,
      "loss": 0.7945,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6182142496109009,
      "rewards/margins": -0.13901755213737488,
      "rewards/rejected": -0.4791966676712036,
      "step": 758
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.9063500315045586e-05,
      "logits/chosen": -2.5884640216827393,
      "logits/rejected": -2.641512393951416,
      "logps/chosen": -186.569091796875,
      "logps/rejected": -200.59988403320312,
      "loss": 0.7447,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6641149520874023,
      "rewards/margins": -0.048228248953819275,
      "rewards/rejected": -0.6158866882324219,
      "step": 759
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.9060673921288716e-05,
      "logits/chosen": -2.633413791656494,
      "logits/rejected": -2.598926305770874,
      "logps/chosen": -161.99928283691406,
      "logps/rejected": -154.25979614257812,
      "loss": 0.7324,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.37947404384613037,
      "rewards/margins": -0.021064819768071175,
      "rewards/rejected": -0.35840916633605957,
      "step": 760
    },
    {
      "epoch": 1.0,
      "learning_rate": 4.905784335052839e-05,
      "logits/chosen": -2.534295082092285,
      "logits/rejected": -2.5714004039764404,
      "logps/chosen": -213.0132598876953,
      "logps/rejected": -223.10958862304688,
      "loss": 0.685,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.7191438674926758,
      "rewards/margins": 0.12883953750133514,
      "rewards/rejected": -0.8479833602905273,
      "step": 761
    },
    {
      "epoch": 1.0,
      "learning_rate": 4.905500860325599e-05,
      "logits/chosen": -2.3805532455444336,
      "logits/rejected": -2.502260208129883,
      "logps/chosen": -187.92947387695312,
      "logps/rejected": -244.21273803710938,
      "loss": 0.5568,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.30298084020614624,
      "rewards/margins": 0.5082517862319946,
      "rewards/rejected": -0.8112326264381409,
      "step": 762
    },
    {
      "epoch": 1.0,
      "learning_rate": 4.905216967996367e-05,
      "logits/chosen": -2.494596242904663,
      "logits/rejected": -2.5522210597991943,
      "logps/chosen": -193.31663513183594,
      "logps/rejected": -253.28793334960938,
      "loss": 0.5326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.31040558218955994,
      "rewards/margins": 0.41559797525405884,
      "rewards/rejected": -0.7260035872459412,
      "step": 763
    },
    {
      "epoch": 1.0,
      "learning_rate": 4.904932658114423e-05,
      "logits/chosen": -2.4357709884643555,
      "logits/rejected": -2.402021884918213,
      "logps/chosen": -207.14736938476562,
      "logps/rejected": -224.48757934570312,
      "loss": 0.7383,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6394506692886353,
      "rewards/margins": 0.015030726790428162,
      "rewards/rejected": -0.6544813513755798,
      "step": 764
    },
    {
      "epoch": 1.0,
      "learning_rate": 4.904647930729128e-05,
      "logits/chosen": -2.492413282394409,
      "logits/rejected": -2.5191187858581543,
      "logps/chosen": -209.2412872314453,
      "logps/rejected": -204.59518432617188,
      "loss": 0.2419,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.42869845032691956,
      "rewards/margins": 2.3007590770721436,
      "rewards/rejected": -1.872060775756836,
      "step": 765
    },
    {
      "epoch": 1.0,
      "learning_rate": 4.9043627858899086e-05,
      "logits/chosen": -2.616361618041992,
      "logits/rejected": -2.640256404876709,
      "logps/chosen": -180.7271270751953,
      "logps/rejected": -231.54254150390625,
      "loss": 0.1455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1313096433877945,
      "rewards/margins": 2.220848321914673,
      "rewards/rejected": -2.08953857421875,
      "step": 766
    },
    {
      "epoch": 1.0,
      "learning_rate": 4.9040772236462695e-05,
      "logits/chosen": -2.3850975036621094,
      "logits/rejected": -2.354506254196167,
      "logps/chosen": -176.04693603515625,
      "logps/rejected": -191.62985229492188,
      "loss": 0.2206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09096850454807281,
      "rewards/margins": 1.8953343629837036,
      "rewards/rejected": -1.8043657541275024,
      "step": 767
    },
    {
      "epoch": 1.01,
      "learning_rate": 4.903791244047783e-05,
      "logits/chosen": -2.448336362838745,
      "logits/rejected": -2.4442625045776367,
      "logps/chosen": -159.71731567382812,
      "logps/rejected": -198.8458251953125,
      "loss": 0.2542,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14124417304992676,
      "rewards/margins": 1.9044493436813354,
      "rewards/rejected": -1.7632049322128296,
      "step": 768
    },
    {
      "epoch": 1.01,
      "learning_rate": 4.9035048471440966e-05,
      "logits/chosen": -2.39510178565979,
      "logits/rejected": -2.3210549354553223,
      "logps/chosen": -146.8027801513672,
      "logps/rejected": -182.59580993652344,
      "loss": 0.2789,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.44124722480773926,
      "rewards/margins": 2.291222095489502,
      "rewards/rejected": -1.8499748706817627,
      "step": 769
    },
    {
      "epoch": 1.01,
      "learning_rate": 4.90321803298493e-05,
      "logits/chosen": -2.610016107559204,
      "logits/rejected": -2.5617635250091553,
      "logps/chosen": -191.46194458007812,
      "logps/rejected": -232.915283203125,
      "loss": 0.1781,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.716643214225769,
      "rewards/margins": 2.4686214923858643,
      "rewards/rejected": -1.7519782781600952,
      "step": 770
    },
    {
      "epoch": 1.01,
      "learning_rate": 4.9029308016200735e-05,
      "logits/chosen": -2.512070655822754,
      "logits/rejected": -2.522047758102417,
      "logps/chosen": -228.5362548828125,
      "logps/rejected": -267.5350341796875,
      "loss": 0.1084,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.7526074051856995,
      "rewards/margins": 3.0486981868743896,
      "rewards/rejected": -2.296090602874756,
      "step": 771
    },
    {
      "epoch": 1.01,
      "learning_rate": 4.902643153099394e-05,
      "logits/chosen": -2.7084665298461914,
      "logits/rejected": -2.702622413635254,
      "logps/chosen": -214.2321319580078,
      "logps/rejected": -234.52236938476562,
      "loss": 0.2319,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.38715142011642456,
      "rewards/margins": 2.1679859161376953,
      "rewards/rejected": -1.780834436416626,
      "step": 772
    },
    {
      "epoch": 1.01,
      "learning_rate": 4.902355087472826e-05,
      "logits/chosen": -2.4830422401428223,
      "logits/rejected": -2.449504852294922,
      "logps/chosen": -157.030517578125,
      "logps/rejected": -182.37347412109375,
      "loss": 0.2089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3868522644042969,
      "rewards/margins": 2.223161220550537,
      "rewards/rejected": -1.8363089561462402,
      "step": 773
    },
    {
      "epoch": 1.01,
      "learning_rate": 4.902066604790379e-05,
      "logits/chosen": -2.387758731842041,
      "logits/rejected": -2.3118813037872314,
      "logps/chosen": -216.08274841308594,
      "logps/rejected": -232.153076171875,
      "loss": 0.3762,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1578804850578308,
      "rewards/margins": 1.8973913192749023,
      "rewards/rejected": -1.7395107746124268,
      "step": 774
    },
    {
      "epoch": 1.01,
      "learning_rate": 4.901777705102135e-05,
      "logits/chosen": -2.4494662284851074,
      "logits/rejected": -2.5127110481262207,
      "logps/chosen": -173.77740478515625,
      "logps/rejected": -214.7391357421875,
      "loss": 0.1939,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14900563657283783,
      "rewards/margins": 2.133729934692383,
      "rewards/rejected": -1.9847242832183838,
      "step": 775
    },
    {
      "epoch": 1.02,
      "learning_rate": 4.901488388458247e-05,
      "logits/chosen": -2.4321305751800537,
      "logits/rejected": -2.449230432510376,
      "logps/chosen": -173.65748596191406,
      "logps/rejected": -176.95252990722656,
      "loss": 0.218,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.25331249833106995,
      "rewards/margins": 2.11264967918396,
      "rewards/rejected": -1.8593370914459229,
      "step": 776
    },
    {
      "epoch": 1.02,
      "learning_rate": 4.901198654908942e-05,
      "logits/chosen": -2.3953897953033447,
      "logits/rejected": -2.4313783645629883,
      "logps/chosen": -207.082275390625,
      "logps/rejected": -261.56427001953125,
      "loss": 0.1348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4747293293476105,
      "rewards/margins": 3.3515007495880127,
      "rewards/rejected": -2.8767709732055664,
      "step": 777
    },
    {
      "epoch": 1.02,
      "learning_rate": 4.9009085045045175e-05,
      "logits/chosen": -2.613166093826294,
      "logits/rejected": -2.6309146881103516,
      "logps/chosen": -218.4897003173828,
      "logps/rejected": -231.17636108398438,
      "loss": 0.2987,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09049064666032791,
      "rewards/margins": 1.798804521560669,
      "rewards/rejected": -1.8892953395843506,
      "step": 778
    },
    {
      "epoch": 1.02,
      "learning_rate": 4.9006179372953466e-05,
      "logits/chosen": -2.4088118076324463,
      "logits/rejected": -2.3893849849700928,
      "logps/chosen": -231.118408203125,
      "logps/rejected": -272.7298278808594,
      "loss": 0.1272,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22063946723937988,
      "rewards/margins": 3.302457332611084,
      "rewards/rejected": -3.0818183422088623,
      "step": 779
    },
    {
      "epoch": 1.02,
      "learning_rate": 4.9003269533318704e-05,
      "logits/chosen": -2.716017961502075,
      "logits/rejected": -2.5699970722198486,
      "logps/chosen": -216.4443817138672,
      "logps/rejected": -191.3568115234375,
      "loss": 0.1168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5278897285461426,
      "rewards/margins": 2.6199638843536377,
      "rewards/rejected": -2.092073917388916,
      "step": 780
    },
    {
      "epoch": 1.02,
      "learning_rate": 4.900035552664605e-05,
      "logits/chosen": -2.5695183277130127,
      "logits/rejected": -2.6082301139831543,
      "logps/chosen": -200.3180389404297,
      "logps/rejected": -233.2794189453125,
      "loss": 0.1978,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.10721993446350098,
      "rewards/margins": 2.7741177082061768,
      "rewards/rejected": -2.666897773742676,
      "step": 781
    },
    {
      "epoch": 1.02,
      "learning_rate": 4.899743735344139e-05,
      "logits/chosen": -2.1258223056793213,
      "logits/rejected": -2.2032220363616943,
      "logps/chosen": -201.60614013671875,
      "logps/rejected": -282.1663818359375,
      "loss": 0.1137,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04111824184656143,
      "rewards/margins": 3.605041027069092,
      "rewards/rejected": -3.56392240524292,
      "step": 782
    },
    {
      "epoch": 1.02,
      "learning_rate": 4.899451501421133e-05,
      "logits/chosen": -2.280001163482666,
      "logits/rejected": -2.454638957977295,
      "logps/chosen": -185.27694702148438,
      "logps/rejected": -216.79037475585938,
      "loss": 0.181,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.409013569355011,
      "rewards/margins": 2.352353572845459,
      "rewards/rejected": -2.7613673210144043,
      "step": 783
    },
    {
      "epoch": 1.03,
      "learning_rate": 4.899158850946319e-05,
      "logits/chosen": -2.4744930267333984,
      "logits/rejected": -2.5907466411590576,
      "logps/chosen": -216.58287048339844,
      "logps/rejected": -261.81353759765625,
      "loss": 0.1848,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11592866480350494,
      "rewards/margins": 2.6159138679504395,
      "rewards/rejected": -2.4999852180480957,
      "step": 784
    },
    {
      "epoch": 1.03,
      "learning_rate": 4.8988657839705024e-05,
      "logits/chosen": -2.6295738220214844,
      "logits/rejected": -2.565519332885742,
      "logps/chosen": -175.5509033203125,
      "logps/rejected": -192.17001342773438,
      "loss": 0.2344,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.13090485334396362,
      "rewards/margins": 2.3108062744140625,
      "rewards/rejected": -2.44171142578125,
      "step": 785
    },
    {
      "epoch": 1.03,
      "learning_rate": 4.89857230054456e-05,
      "logits/chosen": -2.4025466442108154,
      "logits/rejected": -2.478341817855835,
      "logps/chosen": -160.97872924804688,
      "logps/rejected": -221.78729248046875,
      "loss": 0.2185,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18785348534584045,
      "rewards/margins": 2.667365074157715,
      "rewards/rejected": -2.8552186489105225,
      "step": 786
    },
    {
      "epoch": 1.03,
      "learning_rate": 4.898278400719442e-05,
      "logits/chosen": -2.309408664703369,
      "logits/rejected": -2.336895227432251,
      "logps/chosen": -194.12857055664062,
      "logps/rejected": -229.64393615722656,
      "loss": 0.1647,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5826263427734375,
      "rewards/margins": 2.673534870147705,
      "rewards/rejected": -3.2561612129211426,
      "step": 787
    },
    {
      "epoch": 1.03,
      "learning_rate": 4.897984084546169e-05,
      "logits/chosen": -2.5822911262512207,
      "logits/rejected": -2.594147205352783,
      "logps/chosen": -189.2248992919922,
      "logps/rejected": -217.4370880126953,
      "loss": 0.1947,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18159398436546326,
      "rewards/margins": 2.9417152404785156,
      "rewards/rejected": -3.123309373855591,
      "step": 788
    },
    {
      "epoch": 1.03,
      "learning_rate": 4.897689352075837e-05,
      "logits/chosen": -2.395965576171875,
      "logits/rejected": -2.3766961097717285,
      "logps/chosen": -234.74583435058594,
      "logps/rejected": -283.2368469238281,
      "loss": 0.2304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.37718701362609863,
      "rewards/margins": 3.130066394805908,
      "rewards/rejected": -3.507253408432007,
      "step": 789
    },
    {
      "epoch": 1.03,
      "learning_rate": 4.897394203359611e-05,
      "logits/chosen": -2.223119020462036,
      "logits/rejected": -2.1850533485412598,
      "logps/chosen": -163.25404357910156,
      "logps/rejected": -190.954833984375,
      "loss": 0.1986,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.48350968956947327,
      "rewards/margins": 2.4246490001678467,
      "rewards/rejected": -2.908158779144287,
      "step": 790
    },
    {
      "epoch": 1.04,
      "learning_rate": 4.897098638448731e-05,
      "logits/chosen": -2.617220878601074,
      "logits/rejected": -2.6898252964019775,
      "logps/chosen": -191.36790466308594,
      "logps/rejected": -270.26287841796875,
      "loss": 0.1789,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7169384360313416,
      "rewards/margins": 2.659780979156494,
      "rewards/rejected": -3.3767194747924805,
      "step": 791
    },
    {
      "epoch": 1.04,
      "learning_rate": 4.896802657394506e-05,
      "logits/chosen": -2.3435750007629395,
      "logits/rejected": -2.2688729763031006,
      "logps/chosen": -154.95065307617188,
      "logps/rejected": -174.447021484375,
      "loss": 0.2578,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7484995722770691,
      "rewards/margins": 2.27825927734375,
      "rewards/rejected": -3.0267586708068848,
      "step": 792
    },
    {
      "epoch": 1.04,
      "learning_rate": 4.8965062602483205e-05,
      "logits/chosen": -2.4353139400482178,
      "logits/rejected": -2.381152629852295,
      "logps/chosen": -270.47259521484375,
      "logps/rejected": -281.2034606933594,
      "loss": 0.16,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4860559105873108,
      "rewards/margins": 3.1953186988830566,
      "rewards/rejected": -3.6813745498657227,
      "step": 793
    },
    {
      "epoch": 1.04,
      "learning_rate": 4.89620944706163e-05,
      "logits/chosen": -2.391406536102295,
      "logits/rejected": -2.426130533218384,
      "logps/chosen": -167.80072021484375,
      "logps/rejected": -207.9481964111328,
      "loss": 0.1768,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.36457765102386475,
      "rewards/margins": 3.225442409515381,
      "rewards/rejected": -3.590019941329956,
      "step": 794
    },
    {
      "epoch": 1.04,
      "learning_rate": 4.8959122178859616e-05,
      "logits/chosen": -2.5002002716064453,
      "logits/rejected": -2.584379196166992,
      "logps/chosen": -194.40185546875,
      "logps/rejected": -240.38218688964844,
      "loss": 0.1416,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7205836176872253,
      "rewards/margins": 3.2785701751708984,
      "rewards/rejected": -3.9991536140441895,
      "step": 795
    },
    {
      "epoch": 1.04,
      "learning_rate": 4.8956145727729156e-05,
      "logits/chosen": -2.7242343425750732,
      "logits/rejected": -2.7420852184295654,
      "logps/chosen": -193.07855224609375,
      "logps/rejected": -213.01675415039062,
      "loss": 0.2132,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5593734979629517,
      "rewards/margins": 2.7595760822296143,
      "rewards/rejected": -3.3189496994018555,
      "step": 796
    },
    {
      "epoch": 1.04,
      "learning_rate": 4.895316511774164e-05,
      "logits/chosen": -2.5653488636016846,
      "logits/rejected": -2.6561145782470703,
      "logps/chosen": -187.71011352539062,
      "logps/rejected": -234.84353637695312,
      "loss": 0.2078,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.685555100440979,
      "rewards/margins": 2.5933103561401367,
      "rewards/rejected": -3.278865337371826,
      "step": 797
    },
    {
      "epoch": 1.04,
      "learning_rate": 4.895018034941451e-05,
      "logits/chosen": -2.6024954319000244,
      "logits/rejected": -2.519937753677368,
      "logps/chosen": -168.91847229003906,
      "logps/rejected": -224.6294708251953,
      "loss": 0.1913,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.535056471824646,
      "rewards/margins": 3.0812528133392334,
      "rewards/rejected": -3.61630916595459,
      "step": 798
    },
    {
      "epoch": 1.05,
      "learning_rate": 4.894719142326593e-05,
      "logits/chosen": -2.21370005607605,
      "logits/rejected": -2.3001461029052734,
      "logps/chosen": -166.51718139648438,
      "logps/rejected": -224.1083526611328,
      "loss": 0.3661,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7483687996864319,
      "rewards/margins": 2.128541946411133,
      "rewards/rejected": -2.8769102096557617,
      "step": 799
    },
    {
      "epoch": 1.05,
      "learning_rate": 4.894419833981478e-05,
      "logits/chosen": -2.4892656803131104,
      "logits/rejected": -2.568337917327881,
      "logps/chosen": -247.6711883544922,
      "logps/rejected": -264.4457092285156,
      "loss": 0.1815,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5068062543869019,
      "rewards/margins": 3.0614163875579834,
      "rewards/rejected": -3.5682225227355957,
      "step": 800
    },
    {
      "epoch": 1.05,
      "learning_rate": 4.894120109958068e-05,
      "logits/chosen": -2.5446887016296387,
      "logits/rejected": -2.5595788955688477,
      "logps/chosen": -148.94400024414062,
      "logps/rejected": -197.58807373046875,
      "loss": 0.2073,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3782866895198822,
      "rewards/margins": 2.5527689456939697,
      "rewards/rejected": -2.931055784225464,
      "step": 801
    },
    {
      "epoch": 1.05,
      "learning_rate": 4.893819970308394e-05,
      "logits/chosen": -2.6223721504211426,
      "logits/rejected": -2.6008565425872803,
      "logps/chosen": -223.73875427246094,
      "logps/rejected": -270.4544982910156,
      "loss": 0.131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.582880973815918,
      "rewards/margins": 2.6789181232452393,
      "rewards/rejected": -3.2617993354797363,
      "step": 802
    },
    {
      "epoch": 1.05,
      "learning_rate": 4.893519415084564e-05,
      "logits/chosen": -2.576484203338623,
      "logits/rejected": -2.459780216217041,
      "logps/chosen": -176.4852294921875,
      "logps/rejected": -229.3186492919922,
      "loss": 0.1817,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5410618782043457,
      "rewards/margins": 2.697970390319824,
      "rewards/rejected": -3.239032030105591,
      "step": 803
    },
    {
      "epoch": 1.05,
      "learning_rate": 4.8932184443387536e-05,
      "logits/chosen": -2.32121205329895,
      "logits/rejected": -2.379164695739746,
      "logps/chosen": -186.08949279785156,
      "logps/rejected": -203.67755126953125,
      "loss": 0.3161,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.43141627311706543,
      "rewards/margins": 2.3219566345214844,
      "rewards/rejected": -2.7533726692199707,
      "step": 804
    },
    {
      "epoch": 1.05,
      "learning_rate": 4.892917058123212e-05,
      "logits/chosen": -2.4327099323272705,
      "logits/rejected": -2.4539988040924072,
      "logps/chosen": -223.7473602294922,
      "logps/rejected": -243.0752410888672,
      "loss": 0.2913,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5450294017791748,
      "rewards/margins": 1.9214046001434326,
      "rewards/rejected": -2.4664340019226074,
      "step": 805
    },
    {
      "epoch": 1.05,
      "learning_rate": 4.892615256490262e-05,
      "logits/chosen": -2.549992084503174,
      "logits/rejected": -2.67326283454895,
      "logps/chosen": -192.02581787109375,
      "logps/rejected": -212.2666778564453,
      "loss": 0.2081,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1347290426492691,
      "rewards/margins": 2.5369441509246826,
      "rewards/rejected": -2.671673059463501,
      "step": 806
    },
    {
      "epoch": 1.06,
      "learning_rate": 4.8923130394922947e-05,
      "logits/chosen": -2.503438949584961,
      "logits/rejected": -2.5232760906219482,
      "logps/chosen": -202.92604064941406,
      "logps/rejected": -230.6805877685547,
      "loss": 0.2341,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3751166760921478,
      "rewards/margins": 2.6738672256469727,
      "rewards/rejected": -3.0489840507507324,
      "step": 807
    },
    {
      "epoch": 1.06,
      "learning_rate": 4.8920104071817786e-05,
      "logits/chosen": -2.3755786418914795,
      "logits/rejected": -2.5557618141174316,
      "logps/chosen": -137.71795654296875,
      "logps/rejected": -216.0382080078125,
      "loss": 0.1602,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.13777978718280792,
      "rewards/margins": 3.1972692012786865,
      "rewards/rejected": -3.3350489139556885,
      "step": 808
    },
    {
      "epoch": 1.06,
      "learning_rate": 4.891707359611251e-05,
      "logits/chosen": -2.4458518028259277,
      "logits/rejected": -2.485722303390503,
      "logps/chosen": -232.0552520751953,
      "logps/rejected": -290.8924255371094,
      "loss": 0.112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5124866962432861,
      "rewards/margins": 3.1787357330322266,
      "rewards/rejected": -3.6912224292755127,
      "step": 809
    },
    {
      "epoch": 1.06,
      "learning_rate": 4.89140389683332e-05,
      "logits/chosen": -2.6487104892730713,
      "logits/rejected": -2.691654920578003,
      "logps/chosen": -200.54293823242188,
      "logps/rejected": -267.951416015625,
      "loss": 0.0792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20664259791374207,
      "rewards/margins": 3.448357582092285,
      "rewards/rejected": -3.6550002098083496,
      "step": 810
    },
    {
      "epoch": 1.06,
      "learning_rate": 4.8911000189006694e-05,
      "logits/chosen": -2.47886323928833,
      "logits/rejected": -2.5354814529418945,
      "logps/chosen": -155.83856201171875,
      "logps/rejected": -193.6571502685547,
      "loss": 0.1333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1021997407078743,
      "rewards/margins": 3.2138075828552246,
      "rewards/rejected": -3.111607789993286,
      "step": 811
    },
    {
      "epoch": 1.06,
      "learning_rate": 4.8907957258660534e-05,
      "logits/chosen": -2.288719654083252,
      "logits/rejected": -2.2621853351593018,
      "logps/chosen": -219.19277954101562,
      "logps/rejected": -233.79931640625,
      "loss": 0.291,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2972843647003174,
      "rewards/margins": 2.0650277137756348,
      "rewards/rejected": -2.3623123168945312,
      "step": 812
    },
    {
      "epoch": 1.06,
      "learning_rate": 4.890491017782296e-05,
      "logits/chosen": -2.5341038703918457,
      "logits/rejected": -2.636235475540161,
      "logps/chosen": -166.26109313964844,
      "logps/rejected": -217.66732788085938,
      "loss": 0.2438,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7079929709434509,
      "rewards/margins": 2.6824710369110107,
      "rewards/rejected": -3.3904640674591064,
      "step": 813
    },
    {
      "epoch": 1.07,
      "learning_rate": 4.890185894702298e-05,
      "logits/chosen": -2.5911717414855957,
      "logits/rejected": -2.6073203086853027,
      "logps/chosen": -208.44723510742188,
      "logps/rejected": -287.8471374511719,
      "loss": 0.0952,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16351568698883057,
      "rewards/margins": 3.8280396461486816,
      "rewards/rejected": -3.6645243167877197,
      "step": 814
    },
    {
      "epoch": 1.07,
      "learning_rate": 4.8898803566790296e-05,
      "logits/chosen": -2.3474974632263184,
      "logits/rejected": -2.4600653648376465,
      "logps/chosen": -225.6314697265625,
      "logps/rejected": -247.96067810058594,
      "loss": 0.1813,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11835913360118866,
      "rewards/margins": 3.3629634380340576,
      "rewards/rejected": -3.481323003768921,
      "step": 815
    },
    {
      "epoch": 1.07,
      "learning_rate": 4.889574403765531e-05,
      "logits/chosen": -2.314152717590332,
      "logits/rejected": -2.2856571674346924,
      "logps/chosen": -222.77545166015625,
      "logps/rejected": -240.05615234375,
      "loss": 0.2903,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5903599858283997,
      "rewards/margins": 2.8621745109558105,
      "rewards/rejected": -3.4525346755981445,
      "step": 816
    },
    {
      "epoch": 1.07,
      "learning_rate": 4.889268036014918e-05,
      "logits/chosen": -2.2042934894561768,
      "logits/rejected": -2.252229928970337,
      "logps/chosen": -168.29946899414062,
      "logps/rejected": -220.2432403564453,
      "loss": 0.181,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6510707139968872,
      "rewards/margins": 2.5488698482513428,
      "rewards/rejected": -3.1999404430389404,
      "step": 817
    },
    {
      "epoch": 1.07,
      "learning_rate": 4.888961253480376e-05,
      "logits/chosen": -2.330993890762329,
      "logits/rejected": -2.439598321914673,
      "logps/chosen": -191.20274353027344,
      "logps/rejected": -240.98287963867188,
      "loss": 0.0927,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.08680248260498047,
      "rewards/margins": 3.754706382751465,
      "rewards/rejected": -3.8415091037750244,
      "step": 818
    },
    {
      "epoch": 1.07,
      "learning_rate": 4.8886540562151653e-05,
      "logits/chosen": -2.60939884185791,
      "logits/rejected": -2.64247989654541,
      "logps/chosen": -186.75680541992188,
      "logps/rejected": -217.1247100830078,
      "loss": 0.1396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13618728518486023,
      "rewards/margins": 2.6276023387908936,
      "rewards/rejected": -2.7637898921966553,
      "step": 819
    },
    {
      "epoch": 1.07,
      "learning_rate": 4.8883464442726146e-05,
      "logits/chosen": -2.451847791671753,
      "logits/rejected": -2.550415277481079,
      "logps/chosen": -194.68426513671875,
      "logps/rejected": -221.12335205078125,
      "loss": 0.2368,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.38820838928222656,
      "rewards/margins": 2.107210159301758,
      "rewards/rejected": -2.4954187870025635,
      "step": 820
    },
    {
      "epoch": 1.07,
      "learning_rate": 4.888038417706126e-05,
      "logits/chosen": -2.4475693702697754,
      "logits/rejected": -2.4619081020355225,
      "logps/chosen": -185.6713409423828,
      "logps/rejected": -246.03526306152344,
      "loss": 0.1785,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.18588271737098694,
      "rewards/margins": 3.4940121173858643,
      "rewards/rejected": -3.6798949241638184,
      "step": 821
    },
    {
      "epoch": 1.08,
      "learning_rate": 4.887729976569174e-05,
      "logits/chosen": -2.6383659839630127,
      "logits/rejected": -2.641087293624878,
      "logps/chosen": -220.7265625,
      "logps/rejected": -246.0904083251953,
      "loss": 0.1677,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5302943587303162,
      "rewards/margins": 3.091730833053589,
      "rewards/rejected": -3.6220250129699707,
      "step": 822
    },
    {
      "epoch": 1.08,
      "learning_rate": 4.8874211209153066e-05,
      "logits/chosen": -2.7357144355773926,
      "logits/rejected": -2.756209373474121,
      "logps/chosen": -228.9771270751953,
      "logps/rejected": -244.65692138671875,
      "loss": 0.1871,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2874932289123535,
      "rewards/margins": 3.040170669555664,
      "rewards/rejected": -3.3276634216308594,
      "step": 823
    },
    {
      "epoch": 1.08,
      "learning_rate": 4.8871118507981396e-05,
      "logits/chosen": -2.541903257369995,
      "logits/rejected": -2.642256259918213,
      "logps/chosen": -172.7350311279297,
      "logps/rejected": -263.6834716796875,
      "loss": 0.1574,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.31187328696250916,
      "rewards/margins": 3.909829616546631,
      "rewards/rejected": -4.221703052520752,
      "step": 824
    },
    {
      "epoch": 1.08,
      "learning_rate": 4.886802166271364e-05,
      "logits/chosen": -2.5496389865875244,
      "logits/rejected": -2.632146120071411,
      "logps/chosen": -189.05809020996094,
      "logps/rejected": -210.13885498046875,
      "loss": 0.15,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08079969137907028,
      "rewards/margins": 3.155087947845459,
      "rewards/rejected": -3.2358880043029785,
      "step": 825
    },
    {
      "epoch": 1.08,
      "learning_rate": 4.8864920673887434e-05,
      "logits/chosen": -2.508222818374634,
      "logits/rejected": -2.495868682861328,
      "logps/chosen": -178.13548278808594,
      "logps/rejected": -204.01275634765625,
      "loss": 0.1071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.022821810096502304,
      "rewards/margins": 3.6802027225494385,
      "rewards/rejected": -3.703024387359619,
      "step": 826
    },
    {
      "epoch": 1.08,
      "learning_rate": 4.88618155420411e-05,
      "logits/chosen": -2.3088462352752686,
      "logits/rejected": -2.442979574203491,
      "logps/chosen": -207.34799194335938,
      "logps/rejected": -256.6688232421875,
      "loss": 0.1586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.63365638256073,
      "rewards/margins": 2.7970571517944336,
      "rewards/rejected": -3.430713415145874,
      "step": 827
    },
    {
      "epoch": 1.08,
      "learning_rate": 4.8858706267713704e-05,
      "logits/chosen": -2.532752275466919,
      "logits/rejected": -2.47641658782959,
      "logps/chosen": -181.96983337402344,
      "logps/rejected": -179.16371154785156,
      "loss": 0.2413,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7146714329719543,
      "rewards/margins": 2.151493787765503,
      "rewards/rejected": -2.8661651611328125,
      "step": 828
    },
    {
      "epoch": 1.08,
      "learning_rate": 4.885559285144503e-05,
      "logits/chosen": -2.4036006927490234,
      "logits/rejected": -2.451720952987671,
      "logps/chosen": -179.5876922607422,
      "logps/rejected": -236.59486389160156,
      "loss": 0.1633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3670279383659363,
      "rewards/margins": 3.0567290782928467,
      "rewards/rejected": -3.4237565994262695,
      "step": 829
    },
    {
      "epoch": 1.09,
      "learning_rate": 4.885247529377557e-05,
      "logits/chosen": -2.4562742710113525,
      "logits/rejected": -2.516852855682373,
      "logps/chosen": -139.80506896972656,
      "logps/rejected": -169.3076171875,
      "loss": 0.2494,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.586427628993988,
      "rewards/margins": 1.9348812103271484,
      "rewards/rejected": -2.521308660507202,
      "step": 830
    },
    {
      "epoch": 1.09,
      "learning_rate": 4.884935359524655e-05,
      "logits/chosen": -2.433465003967285,
      "logits/rejected": -2.401857852935791,
      "logps/chosen": -181.75381469726562,
      "logps/rejected": -226.99884033203125,
      "loss": 0.1503,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.24370934069156647,
      "rewards/margins": 3.303581953048706,
      "rewards/rejected": -3.5472910404205322,
      "step": 831
    },
    {
      "epoch": 1.09,
      "learning_rate": 4.884622775639989e-05,
      "logits/chosen": -2.4640562534332275,
      "logits/rejected": -2.4529128074645996,
      "logps/chosen": -228.76239013671875,
      "logps/rejected": -277.97503662109375,
      "loss": 0.1871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40687793493270874,
      "rewards/margins": 2.9247701168060303,
      "rewards/rejected": -3.331648111343384,
      "step": 832
    },
    {
      "epoch": 1.09,
      "learning_rate": 4.884309777777826e-05,
      "logits/chosen": -2.600520372390747,
      "logits/rejected": -2.6682722568511963,
      "logps/chosen": -239.098388671875,
      "logps/rejected": -298.6829528808594,
      "loss": 0.0985,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.40839752554893494,
      "rewards/margins": 3.9475584030151367,
      "rewards/rejected": -4.355956077575684,
      "step": 833
    },
    {
      "epoch": 1.09,
      "learning_rate": 4.883996365992502e-05,
      "logits/chosen": -2.470278024673462,
      "logits/rejected": -2.446775197982788,
      "logps/chosen": -225.15782165527344,
      "logps/rejected": -273.9099426269531,
      "loss": 0.1943,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9289353489875793,
      "rewards/margins": 3.2997019290924072,
      "rewards/rejected": -4.228637218475342,
      "step": 834
    },
    {
      "epoch": 1.09,
      "learning_rate": 4.883682540338428e-05,
      "logits/chosen": -2.265939474105835,
      "logits/rejected": -2.3391735553741455,
      "logps/chosen": -203.70504760742188,
      "logps/rejected": -275.56427001953125,
      "loss": 0.1327,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4642513394355774,
      "rewards/margins": 4.042669296264648,
      "rewards/rejected": -4.506920337677002,
      "step": 835
    },
    {
      "epoch": 1.09,
      "learning_rate": 4.883368300870083e-05,
      "logits/chosen": -2.431640148162842,
      "logits/rejected": -2.453253984451294,
      "logps/chosen": -199.0352020263672,
      "logps/rejected": -255.3938751220703,
      "loss": 0.1519,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.729556679725647,
      "rewards/margins": 3.61323881149292,
      "rewards/rejected": -4.3427958488464355,
      "step": 836
    },
    {
      "epoch": 1.1,
      "learning_rate": 4.883053647642022e-05,
      "logits/chosen": -2.531667947769165,
      "logits/rejected": -2.473618507385254,
      "logps/chosen": -258.14215087890625,
      "logps/rejected": -253.9119110107422,
      "loss": 0.05,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14007891714572906,
      "rewards/margins": 4.125094890594482,
      "rewards/rejected": -4.26517391204834,
      "step": 837
    },
    {
      "epoch": 1.1,
      "learning_rate": 4.8827385807088674e-05,
      "logits/chosen": -2.368967056274414,
      "logits/rejected": -2.3205454349517822,
      "logps/chosen": -237.3386688232422,
      "logps/rejected": -306.61737060546875,
      "loss": 0.1212,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8273277282714844,
      "rewards/margins": 3.641183614730835,
      "rewards/rejected": -4.468511581420898,
      "step": 838
    },
    {
      "epoch": 1.1,
      "learning_rate": 4.8824231001253186e-05,
      "logits/chosen": -2.4554178714752197,
      "logits/rejected": -2.457733631134033,
      "logps/chosen": -219.16522216796875,
      "logps/rejected": -298.85003662109375,
      "loss": 0.2137,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3690099716186523,
      "rewards/margins": 2.852839231491089,
      "rewards/rejected": -4.221848964691162,
      "step": 839
    },
    {
      "epoch": 1.1,
      "learning_rate": 4.882107205946142e-05,
      "logits/chosen": -2.19034743309021,
      "logits/rejected": -2.1936280727386475,
      "logps/chosen": -196.35073852539062,
      "logps/rejected": -266.54638671875,
      "loss": 0.2458,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5001093149185181,
      "rewards/margins": 3.14082670211792,
      "rewards/rejected": -3.6409358978271484,
      "step": 840
    },
    {
      "epoch": 1.1,
      "learning_rate": 4.8817908982261765e-05,
      "logits/chosen": -2.3098065853118896,
      "logits/rejected": -2.4067697525024414,
      "logps/chosen": -197.4673614501953,
      "logps/rejected": -242.01669311523438,
      "loss": 0.1775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7165354490280151,
      "rewards/margins": 3.300830841064453,
      "rewards/rejected": -4.017366409301758,
      "step": 841
    },
    {
      "epoch": 1.1,
      "learning_rate": 4.881474177020336e-05,
      "logits/chosen": -2.541229009628296,
      "logits/rejected": -2.5617356300354004,
      "logps/chosen": -219.83966064453125,
      "logps/rejected": -247.85601806640625,
      "loss": 0.0592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8507525324821472,
      "rewards/margins": 3.5395710468292236,
      "rewards/rejected": -4.390323638916016,
      "step": 842
    },
    {
      "epoch": 1.1,
      "learning_rate": 4.881157042383605e-05,
      "logits/chosen": -2.2904212474823,
      "logits/rejected": -2.3347108364105225,
      "logps/chosen": -231.440673828125,
      "logps/rejected": -303.37939453125,
      "loss": 0.2497,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1560659408569336,
      "rewards/margins": 2.958097457885742,
      "rewards/rejected": -4.114163398742676,
      "step": 843
    },
    {
      "epoch": 1.1,
      "learning_rate": 4.8808394943710376e-05,
      "logits/chosen": -2.279738187789917,
      "logits/rejected": -2.3323707580566406,
      "logps/chosen": -213.64846801757812,
      "logps/rejected": -239.2349395751953,
      "loss": 0.243,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1719560623168945,
      "rewards/margins": 2.3868768215179443,
      "rewards/rejected": -3.558833122253418,
      "step": 844
    },
    {
      "epoch": 1.11,
      "learning_rate": 4.880521533037762e-05,
      "logits/chosen": -2.3850831985473633,
      "logits/rejected": -2.4819254875183105,
      "logps/chosen": -211.81105041503906,
      "logps/rejected": -253.8992919921875,
      "loss": 0.1005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6735836267471313,
      "rewards/margins": 3.870424509048462,
      "rewards/rejected": -4.544007778167725,
      "step": 845
    },
    {
      "epoch": 1.11,
      "learning_rate": 4.880203158438975e-05,
      "logits/chosen": -2.4267005920410156,
      "logits/rejected": -2.4583656787872314,
      "logps/chosen": -185.1043243408203,
      "logps/rejected": -223.85244750976562,
      "loss": 0.1452,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.911338746547699,
      "rewards/margins": 3.077333688735962,
      "rewards/rejected": -3.9886722564697266,
      "step": 846
    },
    {
      "epoch": 1.11,
      "learning_rate": 4.87988437062995e-05,
      "logits/chosen": -2.0992445945739746,
      "logits/rejected": -2.171211004257202,
      "logps/chosen": -159.3900909423828,
      "logps/rejected": -242.703369140625,
      "loss": 0.25,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7375649213790894,
      "rewards/margins": 3.8496501445770264,
      "rewards/rejected": -4.587214946746826,
      "step": 847
    },
    {
      "epoch": 1.11,
      "learning_rate": 4.879565169666028e-05,
      "logits/chosen": -2.2952377796173096,
      "logits/rejected": -2.3588685989379883,
      "logps/chosen": -201.49908447265625,
      "logps/rejected": -241.18902587890625,
      "loss": 0.2027,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5940706729888916,
      "rewards/margins": 3.3175528049468994,
      "rewards/rejected": -3.91162371635437,
      "step": 848
    },
    {
      "epoch": 1.11,
      "learning_rate": 4.879245555602624e-05,
      "logits/chosen": -1.9072455167770386,
      "logits/rejected": -2.0710573196411133,
      "logps/chosen": -167.665283203125,
      "logps/rejected": -231.74786376953125,
      "loss": 0.2365,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0657278299331665,
      "rewards/margins": 3.313453435897827,
      "rewards/rejected": -4.379181385040283,
      "step": 849
    },
    {
      "epoch": 1.11,
      "learning_rate": 4.878925528495223e-05,
      "logits/chosen": -2.076167106628418,
      "logits/rejected": -2.0585241317749023,
      "logps/chosen": -160.9062957763672,
      "logps/rejected": -198.1348419189453,
      "loss": 0.2318,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9388778209686279,
      "rewards/margins": 2.9717299938201904,
      "rewards/rejected": -3.9106080532073975,
      "step": 850
    },
    {
      "epoch": 1.11,
      "learning_rate": 4.878605088399384e-05,
      "logits/chosen": -2.1791696548461914,
      "logits/rejected": -2.2479426860809326,
      "logps/chosen": -191.9034881591797,
      "logps/rejected": -257.7882995605469,
      "loss": 0.0744,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5156456828117371,
      "rewards/margins": 3.969588041305542,
      "rewards/rejected": -4.485233783721924,
      "step": 851
    },
    {
      "epoch": 1.12,
      "learning_rate": 4.878284235370735e-05,
      "logits/chosen": -2.1952590942382812,
      "logits/rejected": -2.3678793907165527,
      "logps/chosen": -195.3539276123047,
      "logps/rejected": -240.06231689453125,
      "loss": 0.1083,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9977157115936279,
      "rewards/margins": 3.5119776725769043,
      "rewards/rejected": -4.509693622589111,
      "step": 852
    },
    {
      "epoch": 1.12,
      "learning_rate": 4.877962969464978e-05,
      "logits/chosen": -2.0996241569519043,
      "logits/rejected": -2.2190651893615723,
      "logps/chosen": -184.02914428710938,
      "logps/rejected": -213.2720489501953,
      "loss": 0.2316,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.107424020767212,
      "rewards/margins": 2.9243013858795166,
      "rewards/rejected": -4.0317254066467285,
      "step": 853
    },
    {
      "epoch": 1.12,
      "learning_rate": 4.877641290737884e-05,
      "logits/chosen": -2.398710012435913,
      "logits/rejected": -2.455815315246582,
      "logps/chosen": -196.89927673339844,
      "logps/rejected": -228.26943969726562,
      "loss": 0.2229,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7392411828041077,
      "rewards/margins": 3.144235134124756,
      "rewards/rejected": -3.8834760189056396,
      "step": 854
    },
    {
      "epoch": 1.12,
      "learning_rate": 4.8773191992453e-05,
      "logits/chosen": -2.585550546646118,
      "logits/rejected": -2.6128385066986084,
      "logps/chosen": -221.81942749023438,
      "logps/rejected": -235.17910766601562,
      "loss": 0.1117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5257879495620728,
      "rewards/margins": 3.225809335708618,
      "rewards/rejected": -3.7515969276428223,
      "step": 855
    },
    {
      "epoch": 1.12,
      "learning_rate": 4.87699669504314e-05,
      "logits/chosen": -2.3643717765808105,
      "logits/rejected": -2.397857427597046,
      "logps/chosen": -220.61892700195312,
      "logps/rejected": -270.6251220703125,
      "loss": 0.1944,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2166273593902588,
      "rewards/margins": 3.119513511657715,
      "rewards/rejected": -4.3361406326293945,
      "step": 856
    },
    {
      "epoch": 1.12,
      "learning_rate": 4.876673778187393e-05,
      "logits/chosen": -2.182027578353882,
      "logits/rejected": -2.232729434967041,
      "logps/chosen": -172.77215576171875,
      "logps/rejected": -192.00790405273438,
      "loss": 0.2663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9920349717140198,
      "rewards/margins": 3.2848072052001953,
      "rewards/rejected": -4.2768425941467285,
      "step": 857
    },
    {
      "epoch": 1.12,
      "learning_rate": 4.876350448734117e-05,
      "logits/chosen": -2.317965507507324,
      "logits/rejected": -2.4213790893554688,
      "logps/chosen": -170.614990234375,
      "logps/rejected": -179.1256866455078,
      "loss": 0.3067,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.353677749633789,
      "rewards/margins": 1.795522928237915,
      "rewards/rejected": -3.149200916290283,
      "step": 858
    },
    {
      "epoch": 1.12,
      "learning_rate": 4.876026706739444e-05,
      "logits/chosen": -2.362269401550293,
      "logits/rejected": -2.385591506958008,
      "logps/chosen": -277.1317138671875,
      "logps/rejected": -302.71575927734375,
      "loss": 0.2337,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7664923667907715,
      "rewards/margins": 2.5697085857391357,
      "rewards/rejected": -4.336201190948486,
      "step": 859
    },
    {
      "epoch": 1.13,
      "learning_rate": 4.8757025522595756e-05,
      "logits/chosen": -2.5295486450195312,
      "logits/rejected": -2.5323615074157715,
      "logps/chosen": -205.54013061523438,
      "logps/rejected": -218.6345977783203,
      "loss": 0.1368,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2794644236564636,
      "rewards/margins": 2.9989418983459473,
      "rewards/rejected": -3.2784063816070557,
      "step": 860
    },
    {
      "epoch": 1.13,
      "learning_rate": 4.8753779853507874e-05,
      "logits/chosen": -2.117382764816284,
      "logits/rejected": -2.120049238204956,
      "logps/chosen": -196.75279235839844,
      "logps/rejected": -217.56846618652344,
      "loss": 0.1823,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5300887227058411,
      "rewards/margins": 3.763706922531128,
      "rewards/rejected": -4.293795585632324,
      "step": 861
    },
    {
      "epoch": 1.13,
      "learning_rate": 4.8750530060694236e-05,
      "logits/chosen": -1.7907865047454834,
      "logits/rejected": -1.8680351972579956,
      "logps/chosen": -134.9779510498047,
      "logps/rejected": -194.37901306152344,
      "loss": 0.1639,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8513069152832031,
      "rewards/margins": 2.7600948810577393,
      "rewards/rejected": -3.6114020347595215,
      "step": 862
    },
    {
      "epoch": 1.13,
      "learning_rate": 4.874727614471903e-05,
      "logits/chosen": -2.110868453979492,
      "logits/rejected": -2.1362719535827637,
      "logps/chosen": -194.04483032226562,
      "logps/rejected": -234.01123046875,
      "loss": 0.2074,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6307097673416138,
      "rewards/margins": 3.406496047973633,
      "rewards/rejected": -5.037205696105957,
      "step": 863
    },
    {
      "epoch": 1.13,
      "learning_rate": 4.8744018106147135e-05,
      "logits/chosen": -2.32252836227417,
      "logits/rejected": -2.344344139099121,
      "logps/chosen": -224.74435424804688,
      "logps/rejected": -252.48287963867188,
      "loss": 0.1992,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4851446151733398,
      "rewards/margins": 3.0550522804260254,
      "rewards/rejected": -4.540196418762207,
      "step": 864
    },
    {
      "epoch": 1.13,
      "learning_rate": 4.8740755945544156e-05,
      "logits/chosen": -2.2035303115844727,
      "logits/rejected": -2.198829174041748,
      "logps/chosen": -147.6013641357422,
      "logps/rejected": -178.75584411621094,
      "loss": 0.218,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2339822053909302,
      "rewards/margins": 2.6723711490631104,
      "rewards/rejected": -3.90635347366333,
      "step": 865
    },
    {
      "epoch": 1.13,
      "learning_rate": 4.873748966347642e-05,
      "logits/chosen": -2.1133739948272705,
      "logits/rejected": -2.088212251663208,
      "logps/chosen": -183.02053833007812,
      "logps/rejected": -214.74398803710938,
      "loss": 0.1746,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.064955234527588,
      "rewards/margins": 3.2688629627227783,
      "rewards/rejected": -4.333818435668945,
      "step": 866
    },
    {
      "epoch": 1.13,
      "learning_rate": 4.873421926051095e-05,
      "logits/chosen": -2.3783798217773438,
      "logits/rejected": -2.4469428062438965,
      "logps/chosen": -175.8081817626953,
      "logps/rejected": -228.7090301513672,
      "loss": 0.2164,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9884763956069946,
      "rewards/margins": 3.5400943756103516,
      "rewards/rejected": -4.528570652008057,
      "step": 867
    },
    {
      "epoch": 1.14,
      "learning_rate": 4.873094473721552e-05,
      "logits/chosen": -2.122225284576416,
      "logits/rejected": -2.229064464569092,
      "logps/chosen": -163.4876708984375,
      "logps/rejected": -226.0997314453125,
      "loss": 0.1045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.584133267402649,
      "rewards/margins": 3.3204777240753174,
      "rewards/rejected": -4.904611110687256,
      "step": 868
    },
    {
      "epoch": 1.14,
      "learning_rate": 4.872766609415857e-05,
      "logits/chosen": -2.342576265335083,
      "logits/rejected": -2.389909029006958,
      "logps/chosen": -161.72445678710938,
      "logps/rejected": -186.76344299316406,
      "loss": 0.2182,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8539198040962219,
      "rewards/margins": 2.9423582553863525,
      "rewards/rejected": -3.7962779998779297,
      "step": 869
    },
    {
      "epoch": 1.14,
      "learning_rate": 4.872438333190931e-05,
      "logits/chosen": -2.137856960296631,
      "logits/rejected": -2.179835081100464,
      "logps/chosen": -191.9931182861328,
      "logps/rejected": -224.08697509765625,
      "loss": 0.1825,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.108015775680542,
      "rewards/margins": 3.190267562866211,
      "rewards/rejected": -4.298283100128174,
      "step": 870
    },
    {
      "epoch": 1.14,
      "learning_rate": 4.872109645103762e-05,
      "logits/chosen": -2.0209460258483887,
      "logits/rejected": -2.05436372756958,
      "logps/chosen": -242.2396697998047,
      "logps/rejected": -256.2543640136719,
      "loss": 0.233,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9021006226539612,
      "rewards/margins": 2.7395806312561035,
      "rewards/rejected": -3.64168119430542,
      "step": 871
    },
    {
      "epoch": 1.14,
      "learning_rate": 4.871780545211411e-05,
      "logits/chosen": -2.396819829940796,
      "logits/rejected": -2.3696892261505127,
      "logps/chosen": -262.4898376464844,
      "logps/rejected": -300.5354919433594,
      "loss": 0.3388,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.824455976486206,
      "rewards/margins": 3.6746246814727783,
      "rewards/rejected": -4.499080657958984,
      "step": 872
    },
    {
      "epoch": 1.14,
      "learning_rate": 4.8714510335710114e-05,
      "logits/chosen": -2.2704551219940186,
      "logits/rejected": -2.289527416229248,
      "logps/chosen": -190.1211395263672,
      "logps/rejected": -210.47433471679688,
      "loss": 0.2625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8054894208908081,
      "rewards/margins": 3.2580995559692383,
      "rewards/rejected": -4.063589096069336,
      "step": 873
    },
    {
      "epoch": 1.14,
      "learning_rate": 4.871121110239768e-05,
      "logits/chosen": -2.0636115074157715,
      "logits/rejected": -2.149376392364502,
      "logps/chosen": -178.03355407714844,
      "logps/rejected": -191.28359985351562,
      "loss": 0.1928,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7658480405807495,
      "rewards/margins": 3.3343939781188965,
      "rewards/rejected": -4.100241661071777,
      "step": 874
    },
    {
      "epoch": 1.15,
      "learning_rate": 4.870790775274955e-05,
      "logits/chosen": -2.3600223064422607,
      "logits/rejected": -2.313652276992798,
      "logps/chosen": -202.82212829589844,
      "logps/rejected": -241.97523498535156,
      "loss": 0.1404,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3490352630615234,
      "rewards/margins": 3.479907512664795,
      "rewards/rejected": -4.828942775726318,
      "step": 875
    },
    {
      "epoch": 1.15,
      "learning_rate": 4.8704600287339205e-05,
      "logits/chosen": -2.272904396057129,
      "logits/rejected": -2.3197202682495117,
      "logps/chosen": -195.68267822265625,
      "logps/rejected": -230.81982421875,
      "loss": 0.1807,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.507330060005188,
      "rewards/margins": 3.294886827468872,
      "rewards/rejected": -4.802216529846191,
      "step": 876
    },
    {
      "epoch": 1.15,
      "learning_rate": 4.870128870674082e-05,
      "logits/chosen": -2.062432289123535,
      "logits/rejected": -2.095883846282959,
      "logps/chosen": -177.28150939941406,
      "logps/rejected": -238.36048889160156,
      "loss": 0.0998,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5119123458862305,
      "rewards/margins": 3.5114619731903076,
      "rewards/rejected": -5.023374080657959,
      "step": 877
    },
    {
      "epoch": 1.15,
      "learning_rate": 4.869797301152931e-05,
      "logits/chosen": -2.322573184967041,
      "logits/rejected": -2.352694511413574,
      "logps/chosen": -171.1867218017578,
      "logps/rejected": -209.19857788085938,
      "loss": 0.1745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4840812683105469,
      "rewards/margins": 3.0589983463287354,
      "rewards/rejected": -4.543079376220703,
      "step": 878
    },
    {
      "epoch": 1.15,
      "learning_rate": 4.869465320228028e-05,
      "logits/chosen": -1.5731091499328613,
      "logits/rejected": -1.5635374784469604,
      "logps/chosen": -162.3439483642578,
      "logps/rejected": -218.9515380859375,
      "loss": 0.1648,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3453707695007324,
      "rewards/margins": 3.3265371322631836,
      "rewards/rejected": -4.671907901763916,
      "step": 879
    },
    {
      "epoch": 1.15,
      "learning_rate": 4.869132927957007e-05,
      "logits/chosen": -2.057164192199707,
      "logits/rejected": -2.169400930404663,
      "logps/chosen": -146.08116149902344,
      "logps/rejected": -217.79507446289062,
      "loss": 0.1879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3128044605255127,
      "rewards/margins": 4.181766033172607,
      "rewards/rejected": -5.494570732116699,
      "step": 880
    },
    {
      "epoch": 1.15,
      "learning_rate": 4.86880012439757e-05,
      "logits/chosen": -2.09201979637146,
      "logits/rejected": -2.123706817626953,
      "logps/chosen": -205.8859100341797,
      "logps/rejected": -238.71531677246094,
      "loss": 0.2388,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5409486293792725,
      "rewards/margins": 2.8045778274536133,
      "rewards/rejected": -4.345526695251465,
      "step": 881
    },
    {
      "epoch": 1.15,
      "learning_rate": 4.868466909607494e-05,
      "logits/chosen": -1.9597307443618774,
      "logits/rejected": -2.0664074420928955,
      "logps/chosen": -208.81736755371094,
      "logps/rejected": -236.41000366210938,
      "loss": 0.0614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3827075958251953,
      "rewards/margins": 3.953075647354126,
      "rewards/rejected": -5.3357834815979,
      "step": 882
    },
    {
      "epoch": 1.16,
      "learning_rate": 4.868133283644627e-05,
      "logits/chosen": -2.1879220008850098,
      "logits/rejected": -2.2641048431396484,
      "logps/chosen": -195.26947021484375,
      "logps/rejected": -233.18475341796875,
      "loss": 0.1049,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.485203742980957,
      "rewards/margins": 3.5149152278900146,
      "rewards/rejected": -5.000119209289551,
      "step": 883
    },
    {
      "epoch": 1.16,
      "learning_rate": 4.867799246566887e-05,
      "logits/chosen": -1.873902440071106,
      "logits/rejected": -1.9271022081375122,
      "logps/chosen": -218.30856323242188,
      "logps/rejected": -250.31251525878906,
      "loss": 0.1605,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.570319652557373,
      "rewards/margins": 4.087669372558594,
      "rewards/rejected": -5.657989025115967,
      "step": 884
    },
    {
      "epoch": 1.16,
      "learning_rate": 4.867464798432262e-05,
      "logits/chosen": -2.293184995651245,
      "logits/rejected": -2.348799705505371,
      "logps/chosen": -204.55638122558594,
      "logps/rejected": -239.99803161621094,
      "loss": 0.1635,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4229328632354736,
      "rewards/margins": 4.076297283172607,
      "rewards/rejected": -5.499229907989502,
      "step": 885
    },
    {
      "epoch": 1.16,
      "learning_rate": 4.8671299392988146e-05,
      "logits/chosen": -1.9167914390563965,
      "logits/rejected": -1.835463285446167,
      "logps/chosen": -228.05908203125,
      "logps/rejected": -258.6400146484375,
      "loss": 0.2577,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6862237453460693,
      "rewards/margins": 2.3948185443878174,
      "rewards/rejected": -4.081042289733887,
      "step": 886
    },
    {
      "epoch": 1.16,
      "learning_rate": 4.866794669224678e-05,
      "logits/chosen": -2.263591766357422,
      "logits/rejected": -2.2648370265960693,
      "logps/chosen": -323.7992248535156,
      "logps/rejected": -340.5594177246094,
      "loss": 0.0843,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0585577487945557,
      "rewards/margins": 3.989744186401367,
      "rewards/rejected": -5.048301696777344,
      "step": 887
    },
    {
      "epoch": 1.16,
      "learning_rate": 4.8664589882680544e-05,
      "logits/chosen": -2.3918349742889404,
      "logits/rejected": -2.4805490970611572,
      "logps/chosen": -153.11036682128906,
      "logps/rejected": -231.31317138671875,
      "loss": 0.2822,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8393661975860596,
      "rewards/margins": 3.1571884155273438,
      "rewards/rejected": -3.9965548515319824,
      "step": 888
    },
    {
      "epoch": 1.16,
      "learning_rate": 4.866122896487221e-05,
      "logits/chosen": -2.1840920448303223,
      "logits/rejected": -2.304276466369629,
      "logps/chosen": -167.58294677734375,
      "logps/rejected": -200.28634643554688,
      "loss": 0.1857,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8503859043121338,
      "rewards/margins": 3.1673898696899414,
      "rewards/rejected": -4.017776012420654,
      "step": 889
    },
    {
      "epoch": 1.16,
      "learning_rate": 4.865786393940522e-05,
      "logits/chosen": -2.069397211074829,
      "logits/rejected": -2.1860339641571045,
      "logps/chosen": -179.6785125732422,
      "logps/rejected": -228.13262939453125,
      "loss": 0.0932,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1931277513504028,
      "rewards/margins": 3.2066426277160645,
      "rewards/rejected": -4.399770736694336,
      "step": 890
    },
    {
      "epoch": 1.17,
      "learning_rate": 4.8654494806863775e-05,
      "logits/chosen": -2.1783037185668945,
      "logits/rejected": -2.2106220722198486,
      "logps/chosen": -193.16891479492188,
      "logps/rejected": -243.27835083007812,
      "loss": 0.2,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0907084941864014,
      "rewards/margins": 3.496738910675049,
      "rewards/rejected": -4.587447643280029,
      "step": 891
    },
    {
      "epoch": 1.17,
      "learning_rate": 4.865112156783275e-05,
      "logits/chosen": -2.294567584991455,
      "logits/rejected": -2.1514720916748047,
      "logps/chosen": -214.3270263671875,
      "logps/rejected": -187.45822143554688,
      "loss": 0.214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0721594095230103,
      "rewards/margins": 2.2023167610168457,
      "rewards/rejected": -3.2744765281677246,
      "step": 892
    },
    {
      "epoch": 1.17,
      "learning_rate": 4.864774422289776e-05,
      "logits/chosen": -2.0853824615478516,
      "logits/rejected": -2.1526715755462646,
      "logps/chosen": -204.90089416503906,
      "logps/rejected": -246.01055908203125,
      "loss": 0.112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.11933372169733047,
      "rewards/margins": 4.104992866516113,
      "rewards/rejected": -4.2243266105651855,
      "step": 893
    },
    {
      "epoch": 1.17,
      "learning_rate": 4.864436277264511e-05,
      "logits/chosen": -2.1137301921844482,
      "logits/rejected": -2.1245357990264893,
      "logps/chosen": -159.5576171875,
      "logps/rejected": -167.99693298339844,
      "loss": 0.3612,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1939194202423096,
      "rewards/margins": 2.4540228843688965,
      "rewards/rejected": -3.647942543029785,
      "step": 894
    },
    {
      "epoch": 1.17,
      "learning_rate": 4.864097721766184e-05,
      "logits/chosen": -2.0749716758728027,
      "logits/rejected": -2.199066162109375,
      "logps/chosen": -196.0050048828125,
      "logps/rejected": -234.677001953125,
      "loss": 0.1027,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.997223973274231,
      "rewards/margins": 3.582221269607544,
      "rewards/rejected": -4.5794453620910645,
      "step": 895
    },
    {
      "epoch": 1.17,
      "learning_rate": 4.8637587558535695e-05,
      "logits/chosen": -2.2920432090759277,
      "logits/rejected": -2.3417181968688965,
      "logps/chosen": -236.25619506835938,
      "logps/rejected": -282.4396057128906,
      "loss": 0.2044,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.38354048132896423,
      "rewards/margins": 2.915595531463623,
      "rewards/rejected": -3.29913592338562,
      "step": 896
    },
    {
      "epoch": 1.17,
      "learning_rate": 4.863419379585512e-05,
      "logits/chosen": -2.2515666484832764,
      "logits/rejected": -2.2877609729766846,
      "logps/chosen": -189.43197631835938,
      "logps/rejected": -230.85414123535156,
      "loss": 0.1691,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3132818937301636,
      "rewards/margins": 3.6810238361358643,
      "rewards/rejected": -3.9943056106567383,
      "step": 897
    },
    {
      "epoch": 1.18,
      "learning_rate": 4.863079593020928e-05,
      "logits/chosen": -2.3358492851257324,
      "logits/rejected": -2.2418711185455322,
      "logps/chosen": -200.99757385253906,
      "logps/rejected": -208.28480529785156,
      "loss": 0.1435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17593544721603394,
      "rewards/margins": 3.3050811290740967,
      "rewards/rejected": -3.4810166358947754,
      "step": 898
    },
    {
      "epoch": 1.18,
      "learning_rate": 4.8627393962188075e-05,
      "logits/chosen": -2.2615466117858887,
      "logits/rejected": -2.26615047454834,
      "logps/chosen": -265.6878662109375,
      "logps/rejected": -265.2508239746094,
      "loss": 0.2,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6857917904853821,
      "rewards/margins": 3.3371822834014893,
      "rewards/rejected": -4.022973537445068,
      "step": 899
    },
    {
      "epoch": 1.18,
      "learning_rate": 4.8623987892382066e-05,
      "logits/chosen": -2.3570547103881836,
      "logits/rejected": -2.3362598419189453,
      "logps/chosen": -199.10662841796875,
      "logps/rejected": -217.95907592773438,
      "loss": 0.111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.27905502915382385,
      "rewards/margins": 3.4710071086883545,
      "rewards/rejected": -3.7500622272491455,
      "step": 900
    },
    {
      "epoch": 1.18,
      "learning_rate": 4.862057772138258e-05,
      "logits/chosen": -2.2032244205474854,
      "logits/rejected": -2.2657198905944824,
      "logps/chosen": -213.52206420898438,
      "logps/rejected": -225.04454040527344,
      "loss": 0.1846,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8243767023086548,
      "rewards/margins": 3.046542167663574,
      "rewards/rejected": -3.8709182739257812,
      "step": 901
    },
    {
      "epoch": 1.18,
      "learning_rate": 4.861716344978162e-05,
      "logits/chosen": -2.394089698791504,
      "logits/rejected": -2.292032241821289,
      "logps/chosen": -234.7174835205078,
      "logps/rejected": -255.44989013671875,
      "loss": 0.1544,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5697602033615112,
      "rewards/margins": 3.2541327476501465,
      "rewards/rejected": -3.8238930702209473,
      "step": 902
    },
    {
      "epoch": 1.18,
      "learning_rate": 4.861374507817193e-05,
      "logits/chosen": -2.3428890705108643,
      "logits/rejected": -2.3085713386535645,
      "logps/chosen": -221.3970489501953,
      "logps/rejected": -264.0530090332031,
      "loss": 0.0761,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.48718422651290894,
      "rewards/margins": 3.7521519660949707,
      "rewards/rejected": -4.239336013793945,
      "step": 903
    },
    {
      "epoch": 1.18,
      "learning_rate": 4.8610322607146926e-05,
      "logits/chosen": -2.417067766189575,
      "logits/rejected": -2.393913745880127,
      "logps/chosen": -178.41981506347656,
      "logps/rejected": -186.7463836669922,
      "loss": 0.277,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6322096586227417,
      "rewards/margins": 1.9191558361053467,
      "rewards/rejected": -2.551365375518799,
      "step": 904
    },
    {
      "epoch": 1.18,
      "learning_rate": 4.8606896037300776e-05,
      "logits/chosen": -2.2859747409820557,
      "logits/rejected": -2.3659892082214355,
      "logps/chosen": -210.5970458984375,
      "logps/rejected": -241.645263671875,
      "loss": 0.0721,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2545009255409241,
      "rewards/margins": 3.632350444793701,
      "rewards/rejected": -3.8868515491485596,
      "step": 905
    },
    {
      "epoch": 1.19,
      "learning_rate": 4.860346536922834e-05,
      "logits/chosen": -2.369377374649048,
      "logits/rejected": -2.4897830486297607,
      "logps/chosen": -254.58274841308594,
      "logps/rejected": -319.5621643066406,
      "loss": 0.1598,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3183284401893616,
      "rewards/margins": 3.658812999725342,
      "rewards/rejected": -3.9771416187286377,
      "step": 906
    },
    {
      "epoch": 1.19,
      "learning_rate": 4.860003060352518e-05,
      "logits/chosen": -2.3488306999206543,
      "logits/rejected": -2.3815548419952393,
      "logps/chosen": -171.2317352294922,
      "logps/rejected": -204.7757568359375,
      "loss": 0.1461,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7481203675270081,
      "rewards/margins": 2.8177552223205566,
      "rewards/rejected": -3.56587553024292,
      "step": 907
    },
    {
      "epoch": 1.19,
      "learning_rate": 4.85965917407876e-05,
      "logits/chosen": -2.2379674911499023,
      "logits/rejected": -2.3881618976593018,
      "logps/chosen": -165.37860107421875,
      "logps/rejected": -203.4194793701172,
      "loss": 0.2971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6805337071418762,
      "rewards/margins": 2.6565561294555664,
      "rewards/rejected": -3.337089776992798,
      "step": 908
    },
    {
      "epoch": 1.19,
      "learning_rate": 4.8593148781612587e-05,
      "logits/chosen": -2.630882740020752,
      "logits/rejected": -2.544029951095581,
      "logps/chosen": -201.74972534179688,
      "logps/rejected": -221.31356811523438,
      "loss": 0.1897,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4801185429096222,
      "rewards/margins": 3.1227259635925293,
      "rewards/rejected": -3.602844715118408,
      "step": 909
    },
    {
      "epoch": 1.19,
      "learning_rate": 4.858970172659785e-05,
      "logits/chosen": -2.405344247817993,
      "logits/rejected": -2.471182107925415,
      "logps/chosen": -232.18885803222656,
      "logps/rejected": -307.023681640625,
      "loss": 0.0831,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4412127435207367,
      "rewards/margins": 3.6795432567596436,
      "rewards/rejected": -4.12075662612915,
      "step": 910
    },
    {
      "epoch": 1.19,
      "learning_rate": 4.85862505763418e-05,
      "logits/chosen": -1.9770020246505737,
      "logits/rejected": -1.8537812232971191,
      "logps/chosen": -236.51913452148438,
      "logps/rejected": -222.99220275878906,
      "loss": 0.2417,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4903448224067688,
      "rewards/margins": 2.971850872039795,
      "rewards/rejected": -3.46219539642334,
      "step": 911
    },
    {
      "epoch": 1.19,
      "learning_rate": 4.858279533144358e-05,
      "logits/chosen": -2.5417568683624268,
      "logits/rejected": -2.6035280227661133,
      "logps/chosen": -172.0741729736328,
      "logps/rejected": -214.9192657470703,
      "loss": 0.1949,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9094291925430298,
      "rewards/margins": 2.6431045532226562,
      "rewards/rejected": -3.5525336265563965,
      "step": 912
    },
    {
      "epoch": 1.19,
      "learning_rate": 4.857933599250303e-05,
      "logits/chosen": -2.342681407928467,
      "logits/rejected": -2.256180763244629,
      "logps/chosen": -196.65240478515625,
      "logps/rejected": -195.11648559570312,
      "loss": 0.1316,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5412741899490356,
      "rewards/margins": 3.1819822788238525,
      "rewards/rejected": -3.7232565879821777,
      "step": 913
    },
    {
      "epoch": 1.2,
      "learning_rate": 4.857587256012068e-05,
      "logits/chosen": -2.2213358879089355,
      "logits/rejected": -2.223132371902466,
      "logps/chosen": -217.80227661132812,
      "logps/rejected": -258.8191833496094,
      "loss": 0.2183,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.33522820472717285,
      "rewards/margins": 3.4243385791778564,
      "rewards/rejected": -3.759566307067871,
      "step": 914
    },
    {
      "epoch": 1.2,
      "learning_rate": 4.857240503489782e-05,
      "logits/chosen": -2.633331060409546,
      "logits/rejected": -2.5645124912261963,
      "logps/chosen": -200.99314880371094,
      "logps/rejected": -216.61944580078125,
      "loss": 0.0838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03604121133685112,
      "rewards/margins": 3.366603136062622,
      "rewards/rejected": -3.402644157409668,
      "step": 915
    },
    {
      "epoch": 1.2,
      "learning_rate": 4.8568933417436416e-05,
      "logits/chosen": -2.60661244392395,
      "logits/rejected": -2.568995475769043,
      "logps/chosen": -229.74319458007812,
      "logps/rejected": -272.7928161621094,
      "loss": 0.0913,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10647392272949219,
      "rewards/margins": 4.324003219604492,
      "rewards/rejected": -4.430477142333984,
      "step": 916
    },
    {
      "epoch": 1.2,
      "learning_rate": 4.856545770833913e-05,
      "logits/chosen": -2.1530046463012695,
      "logits/rejected": -2.075482130050659,
      "logps/chosen": -218.10240173339844,
      "logps/rejected": -234.6465301513672,
      "loss": 0.166,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3659510612487793,
      "rewards/margins": 4.201175212860107,
      "rewards/rejected": -3.8352243900299072,
      "step": 917
    },
    {
      "epoch": 1.2,
      "learning_rate": 4.8561977908209384e-05,
      "logits/chosen": -2.322742462158203,
      "logits/rejected": -2.2862117290496826,
      "logps/chosen": -188.68533325195312,
      "logps/rejected": -230.53916931152344,
      "loss": 0.1539,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2586498260498047,
      "rewards/margins": 3.89760684967041,
      "rewards/rejected": -4.156256198883057,
      "step": 918
    },
    {
      "epoch": 1.2,
      "learning_rate": 4.855849401765127e-05,
      "logits/chosen": -2.245067834854126,
      "logits/rejected": -2.264173984527588,
      "logps/chosen": -163.19442749023438,
      "logps/rejected": -244.39859008789062,
      "loss": 0.2118,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5528127551078796,
      "rewards/margins": 2.983473539352417,
      "rewards/rejected": -3.5362865924835205,
      "step": 919
    },
    {
      "epoch": 1.2,
      "learning_rate": 4.8555006037269594e-05,
      "logits/chosen": -2.4369189739227295,
      "logits/rejected": -2.4961891174316406,
      "logps/chosen": -199.59515380859375,
      "logps/rejected": -218.68148803710938,
      "loss": 0.2001,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.507201611995697,
      "rewards/margins": 2.782557487487793,
      "rewards/rejected": -3.2897589206695557,
      "step": 920
    },
    {
      "epoch": 1.21,
      "learning_rate": 4.855151396766988e-05,
      "logits/chosen": -2.557804584503174,
      "logits/rejected": -2.554647922515869,
      "logps/chosen": -195.2904815673828,
      "logps/rejected": -226.03387451171875,
      "loss": 0.1858,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.013599276542663574,
      "rewards/margins": 4.105103492736816,
      "rewards/rejected": -4.091504096984863,
      "step": 921
    },
    {
      "epoch": 1.21,
      "learning_rate": 4.854801780945837e-05,
      "logits/chosen": -1.9899704456329346,
      "logits/rejected": -1.9653894901275635,
      "logps/chosen": -201.0773468017578,
      "logps/rejected": -210.57366943359375,
      "loss": 0.247,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5728083252906799,
      "rewards/margins": 2.8485476970672607,
      "rewards/rejected": -3.421356201171875,
      "step": 922
    },
    {
      "epoch": 1.21,
      "learning_rate": 4.854451756324201e-05,
      "logits/chosen": -2.334390878677368,
      "logits/rejected": -2.2994754314422607,
      "logps/chosen": -231.74093627929688,
      "logps/rejected": -252.7148895263672,
      "loss": 0.2625,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6329984664916992,
      "rewards/margins": 2.753272533416748,
      "rewards/rejected": -3.386270523071289,
      "step": 923
    },
    {
      "epoch": 1.21,
      "learning_rate": 4.854101322962845e-05,
      "logits/chosen": -2.4350640773773193,
      "logits/rejected": -2.4160242080688477,
      "logps/chosen": -202.79510498046875,
      "logps/rejected": -196.56761169433594,
      "loss": 0.1783,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7683102488517761,
      "rewards/margins": 2.6973066329956055,
      "rewards/rejected": -3.4656169414520264,
      "step": 924
    },
    {
      "epoch": 1.21,
      "learning_rate": 4.853750480922604e-05,
      "logits/chosen": -2.4578471183776855,
      "logits/rejected": -2.4201016426086426,
      "logps/chosen": -236.7185516357422,
      "logps/rejected": -225.40760803222656,
      "loss": 0.1667,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3481481373310089,
      "rewards/margins": 3.050295352935791,
      "rewards/rejected": -3.3984434604644775,
      "step": 925
    },
    {
      "epoch": 1.21,
      "learning_rate": 4.853399230264387e-05,
      "logits/chosen": -2.5239510536193848,
      "logits/rejected": -2.5775818824768066,
      "logps/chosen": -139.86842346191406,
      "logps/rejected": -190.04673767089844,
      "loss": 0.3665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7791432738304138,
      "rewards/margins": 1.9640579223632812,
      "rewards/rejected": -2.74320125579834,
      "step": 926
    },
    {
      "epoch": 1.21,
      "learning_rate": 4.853047571049171e-05,
      "logits/chosen": -2.3366544246673584,
      "logits/rejected": -2.2847678661346436,
      "logps/chosen": -184.1388702392578,
      "logps/rejected": -197.55262756347656,
      "loss": 0.1811,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3248026967048645,
      "rewards/margins": 2.802187919616699,
      "rewards/rejected": -3.126990795135498,
      "step": 927
    },
    {
      "epoch": 1.21,
      "learning_rate": 4.852695503338005e-05,
      "logits/chosen": -2.2322957515716553,
      "logits/rejected": -2.232264518737793,
      "logps/chosen": -218.7916259765625,
      "logps/rejected": -239.33642578125,
      "loss": 0.2817,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7828483581542969,
      "rewards/margins": 2.7246387004852295,
      "rewards/rejected": -3.5074868202209473,
      "step": 928
    },
    {
      "epoch": 1.22,
      "learning_rate": 4.85234302719201e-05,
      "logits/chosen": -2.3104605674743652,
      "logits/rejected": -2.292299509048462,
      "logps/chosen": -223.07472229003906,
      "logps/rejected": -220.38296508789062,
      "loss": 0.1399,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13662558794021606,
      "rewards/margins": 3.366118907928467,
      "rewards/rejected": -3.502744436264038,
      "step": 929
    },
    {
      "epoch": 1.22,
      "learning_rate": 4.851990142672376e-05,
      "logits/chosen": -2.27694034576416,
      "logits/rejected": -2.416929244995117,
      "logps/chosen": -163.55599975585938,
      "logps/rejected": -187.94540405273438,
      "loss": 0.1647,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6064982414245605,
      "rewards/margins": 2.823979616165161,
      "rewards/rejected": -3.4304776191711426,
      "step": 930
    },
    {
      "epoch": 1.22,
      "learning_rate": 4.8516368498403654e-05,
      "logits/chosen": -2.1894800662994385,
      "logits/rejected": -2.198641777038574,
      "logps/chosen": -192.32850646972656,
      "logps/rejected": -232.83926391601562,
      "loss": 0.2586,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7106565237045288,
      "rewards/margins": 2.401294231414795,
      "rewards/rejected": -3.1119508743286133,
      "step": 931
    },
    {
      "epoch": 1.22,
      "learning_rate": 4.85128314875731e-05,
      "logits/chosen": -2.2995123863220215,
      "logits/rejected": -2.458733320236206,
      "logps/chosen": -184.31101989746094,
      "logps/rejected": -263.00048828125,
      "loss": 0.0955,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.1651136875152588,
      "rewards/margins": 3.4938466548919678,
      "rewards/rejected": -3.6589601039886475,
      "step": 932
    },
    {
      "epoch": 1.22,
      "learning_rate": 4.850929039484614e-05,
      "logits/chosen": -2.3589913845062256,
      "logits/rejected": -2.4086785316467285,
      "logps/chosen": -147.282470703125,
      "logps/rejected": -214.50318908691406,
      "loss": 0.1216,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.45500433444976807,
      "rewards/margins": 3.236088275909424,
      "rewards/rejected": -3.6910927295684814,
      "step": 933
    },
    {
      "epoch": 1.22,
      "learning_rate": 4.850574522083753e-05,
      "logits/chosen": -2.222720146179199,
      "logits/rejected": -2.3172354698181152,
      "logps/chosen": -180.91482543945312,
      "logps/rejected": -231.47238159179688,
      "loss": 0.1583,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7558724284172058,
      "rewards/margins": 2.776618480682373,
      "rewards/rejected": -3.5324912071228027,
      "step": 934
    },
    {
      "epoch": 1.22,
      "learning_rate": 4.8502195966162694e-05,
      "logits/chosen": -2.328801393508911,
      "logits/rejected": -2.4272334575653076,
      "logps/chosen": -154.9918212890625,
      "logps/rejected": -246.27830505371094,
      "loss": 0.0796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6289770007133484,
      "rewards/margins": 3.6553382873535156,
      "rewards/rejected": -4.28431510925293,
      "step": 935
    },
    {
      "epoch": 1.22,
      "learning_rate": 4.849864263143781e-05,
      "logits/chosen": -2.335493564605713,
      "logits/rejected": -2.3662097454071045,
      "logps/chosen": -163.90878295898438,
      "logps/rejected": -192.3372802734375,
      "loss": 0.2689,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5199804306030273,
      "rewards/margins": 2.6915531158447266,
      "rewards/rejected": -3.211533546447754,
      "step": 936
    },
    {
      "epoch": 1.23,
      "learning_rate": 4.8495085217279755e-05,
      "logits/chosen": -2.3108789920806885,
      "logits/rejected": -2.411635637283325,
      "logps/chosen": -178.46237182617188,
      "logps/rejected": -252.6321563720703,
      "loss": 0.0912,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3370632529258728,
      "rewards/margins": 3.8149256706237793,
      "rewards/rejected": -4.151988506317139,
      "step": 937
    },
    {
      "epoch": 1.23,
      "learning_rate": 4.849152372430609e-05,
      "logits/chosen": -2.3515238761901855,
      "logits/rejected": -2.4710798263549805,
      "logps/chosen": -186.52601623535156,
      "logps/rejected": -236.93490600585938,
      "loss": 0.1544,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.4036705791950226,
      "rewards/margins": 3.555159091949463,
      "rewards/rejected": -3.958829402923584,
      "step": 938
    },
    {
      "epoch": 1.23,
      "learning_rate": 4.848795815313511e-05,
      "logits/chosen": -2.0485763549804688,
      "logits/rejected": -2.0782063007354736,
      "logps/chosen": -190.405517578125,
      "logps/rejected": -239.02328491210938,
      "loss": 0.1048,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.12277688086032867,
      "rewards/margins": 3.6550817489624023,
      "rewards/rejected": -3.7778589725494385,
      "step": 939
    },
    {
      "epoch": 1.23,
      "learning_rate": 4.8484388504385806e-05,
      "logits/chosen": -2.2379536628723145,
      "logits/rejected": -2.215625762939453,
      "logps/chosen": -169.50054931640625,
      "logps/rejected": -222.1637420654297,
      "loss": 0.2537,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6230670809745789,
      "rewards/margins": 2.660698413848877,
      "rewards/rejected": -3.2837653160095215,
      "step": 940
    },
    {
      "epoch": 1.23,
      "learning_rate": 4.8480814778677886e-05,
      "logits/chosen": -2.2735724449157715,
      "logits/rejected": -2.31823468208313,
      "logps/chosen": -181.47183227539062,
      "logps/rejected": -204.40936279296875,
      "loss": 0.2299,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45316997170448303,
      "rewards/margins": 3.5822412967681885,
      "rewards/rejected": -4.035411357879639,
      "step": 941
    },
    {
      "epoch": 1.23,
      "learning_rate": 4.847723697663175e-05,
      "logits/chosen": -2.504059076309204,
      "logits/rejected": -2.55910325050354,
      "logps/chosen": -228.25503540039062,
      "logps/rejected": -302.8750305175781,
      "loss": 0.0823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8190544843673706,
      "rewards/margins": 3.961775541305542,
      "rewards/rejected": -4.780829906463623,
      "step": 942
    },
    {
      "epoch": 1.23,
      "learning_rate": 4.847365509886851e-05,
      "logits/chosen": -2.3396451473236084,
      "logits/rejected": -2.4200096130371094,
      "logps/chosen": -202.72042846679688,
      "logps/rejected": -239.9745635986328,
      "loss": 0.0795,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4111427068710327,
      "rewards/margins": 3.3438093662261963,
      "rewards/rejected": -3.7549519538879395,
      "step": 943
    },
    {
      "epoch": 1.24,
      "learning_rate": 4.847006914601001e-05,
      "logits/chosen": -2.4212615489959717,
      "logits/rejected": -2.48210072517395,
      "logps/chosen": -213.84280395507812,
      "logps/rejected": -260.6656188964844,
      "loss": 0.1249,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8076397776603699,
      "rewards/margins": 3.792834997177124,
      "rewards/rejected": -4.600475311279297,
      "step": 944
    },
    {
      "epoch": 1.24,
      "learning_rate": 4.8466479118678766e-05,
      "logits/chosen": -2.2285969257354736,
      "logits/rejected": -2.315225124359131,
      "logps/chosen": -187.538330078125,
      "logps/rejected": -268.7898254394531,
      "loss": 0.2034,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2221934795379639,
      "rewards/margins": 3.516084671020508,
      "rewards/rejected": -4.738278388977051,
      "step": 945
    },
    {
      "epoch": 1.24,
      "learning_rate": 4.846288501749802e-05,
      "logits/chosen": -2.5156943798065186,
      "logits/rejected": -2.5410516262054443,
      "logps/chosen": -222.49359130859375,
      "logps/rejected": -282.3955993652344,
      "loss": 0.1176,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7577163577079773,
      "rewards/margins": 4.017005920410156,
      "rewards/rejected": -4.774722576141357,
      "step": 946
    },
    {
      "epoch": 1.24,
      "learning_rate": 4.8459286843091724e-05,
      "logits/chosen": -2.2465834617614746,
      "logits/rejected": -2.3958494663238525,
      "logps/chosen": -205.92015075683594,
      "logps/rejected": -250.27149963378906,
      "loss": 0.1785,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3019455671310425,
      "rewards/margins": 3.5620269775390625,
      "rewards/rejected": -4.8639726638793945,
      "step": 947
    },
    {
      "epoch": 1.24,
      "learning_rate": 4.845568459608453e-05,
      "logits/chosen": -2.2198760509490967,
      "logits/rejected": -2.3069236278533936,
      "logps/chosen": -164.4407196044922,
      "logps/rejected": -227.1414794921875,
      "loss": 0.19,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6457600593566895,
      "rewards/margins": 3.604132652282715,
      "rewards/rejected": -4.249892711639404,
      "step": 948
    },
    {
      "epoch": 1.24,
      "learning_rate": 4.8452078277101783e-05,
      "logits/chosen": -2.1330924034118652,
      "logits/rejected": -2.2624382972717285,
      "logps/chosen": -141.55337524414062,
      "logps/rejected": -193.1168670654297,
      "loss": 0.18,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4278246760368347,
      "rewards/margins": 3.105529308319092,
      "rewards/rejected": -3.5333540439605713,
      "step": 949
    },
    {
      "epoch": 1.24,
      "learning_rate": 4.8448467886769585e-05,
      "logits/chosen": -1.9669086933135986,
      "logits/rejected": -2.0849437713623047,
      "logps/chosen": -181.0906982421875,
      "logps/rejected": -237.8538055419922,
      "loss": 0.1504,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0563127994537354,
      "rewards/margins": 3.4143903255462646,
      "rewards/rejected": -4.470703125,
      "step": 950
    },
    {
      "epoch": 1.24,
      "learning_rate": 4.844485342571468e-05,
      "logits/chosen": -1.7133499383926392,
      "logits/rejected": -1.701385259628296,
      "logps/chosen": -169.70852661132812,
      "logps/rejected": -198.37237548828125,
      "loss": 0.4077,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4771201610565186,
      "rewards/margins": 2.8859434127807617,
      "rewards/rejected": -4.363063335418701,
      "step": 951
    },
    {
      "epoch": 1.25,
      "learning_rate": 4.844123489456456e-05,
      "logits/chosen": -2.262361526489258,
      "logits/rejected": -2.4248523712158203,
      "logps/chosen": -198.08021545410156,
      "logps/rejected": -303.7059631347656,
      "loss": 0.1379,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.432203769683838,
      "rewards/margins": 3.657777786254883,
      "rewards/rejected": -5.089981555938721,
      "step": 952
    },
    {
      "epoch": 1.25,
      "learning_rate": 4.843761229394741e-05,
      "logits/chosen": -2.3017332553863525,
      "logits/rejected": -2.291661500930786,
      "logps/chosen": -189.6435089111328,
      "logps/rejected": -246.8826141357422,
      "loss": 0.0758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8683138489723206,
      "rewards/margins": 4.264334678649902,
      "rewards/rejected": -5.132648468017578,
      "step": 953
    },
    {
      "epoch": 1.25,
      "learning_rate": 4.843398562449212e-05,
      "logits/chosen": -1.9980626106262207,
      "logits/rejected": -1.996732473373413,
      "logps/chosen": -190.13116455078125,
      "logps/rejected": -222.26028442382812,
      "loss": 0.2367,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.423780918121338,
      "rewards/margins": 2.6461732387542725,
      "rewards/rejected": -4.069953918457031,
      "step": 954
    },
    {
      "epoch": 1.25,
      "learning_rate": 4.84303548868283e-05,
      "logits/chosen": -2.3841123580932617,
      "logits/rejected": -2.439812660217285,
      "logps/chosen": -231.7444610595703,
      "logps/rejected": -276.234619140625,
      "loss": 0.1215,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5399219989776611,
      "rewards/margins": 3.503429651260376,
      "rewards/rejected": -5.043351650238037,
      "step": 955
    },
    {
      "epoch": 1.25,
      "learning_rate": 4.842672008158625e-05,
      "logits/chosen": -2.028545379638672,
      "logits/rejected": -2.0214412212371826,
      "logps/chosen": -144.1702880859375,
      "logps/rejected": -193.11929321289062,
      "loss": 0.3515,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5417052507400513,
      "rewards/margins": 2.0481746196746826,
      "rewards/rejected": -3.5898799896240234,
      "step": 956
    },
    {
      "epoch": 1.25,
      "learning_rate": 4.842308120939699e-05,
      "logits/chosen": -2.440342426300049,
      "logits/rejected": -2.4507193565368652,
      "logps/chosen": -215.12884521484375,
      "logps/rejected": -226.1699981689453,
      "loss": 0.1973,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6147558689117432,
      "rewards/margins": 3.0818982124328613,
      "rewards/rejected": -4.696654319763184,
      "step": 957
    },
    {
      "epoch": 1.25,
      "learning_rate": 4.8419438270892226e-05,
      "logits/chosen": -2.179013252258301,
      "logits/rejected": -2.0757102966308594,
      "logps/chosen": -157.2231903076172,
      "logps/rejected": -176.61666870117188,
      "loss": 0.2506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5769833326339722,
      "rewards/margins": 2.5216851234436035,
      "rewards/rejected": -4.098668098449707,
      "step": 958
    },
    {
      "epoch": 1.26,
      "learning_rate": 4.8415791266704404e-05,
      "logits/chosen": -2.1697206497192383,
      "logits/rejected": -2.1985995769500732,
      "logps/chosen": -197.36570739746094,
      "logps/rejected": -242.4941864013672,
      "loss": 0.217,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0622727870941162,
      "rewards/margins": 3.3413608074188232,
      "rewards/rejected": -4.4036335945129395,
      "step": 959
    },
    {
      "epoch": 1.26,
      "learning_rate": 4.841214019746663e-05,
      "logits/chosen": -2.132883310317993,
      "logits/rejected": -2.2006301879882812,
      "logps/chosen": -195.13247680664062,
      "logps/rejected": -226.83316040039062,
      "loss": 0.2752,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.512275218963623,
      "rewards/margins": 2.76702880859375,
      "rewards/rejected": -4.279304027557373,
      "step": 960
    },
    {
      "epoch": 1.26,
      "learning_rate": 4.840848506381276e-05,
      "logits/chosen": -2.286079168319702,
      "logits/rejected": -2.3662781715393066,
      "logps/chosen": -201.76133728027344,
      "logps/rejected": -226.46603393554688,
      "loss": 0.1871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.301385521888733,
      "rewards/margins": 3.0335183143615723,
      "rewards/rejected": -4.334904193878174,
      "step": 961
    },
    {
      "epoch": 1.26,
      "learning_rate": 4.840482586637732e-05,
      "logits/chosen": -2.354762554168701,
      "logits/rejected": -2.3351876735687256,
      "logps/chosen": -213.7999267578125,
      "logps/rejected": -268.6205139160156,
      "loss": 0.3769,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8948153257369995,
      "rewards/margins": 2.8223719596862793,
      "rewards/rejected": -4.71718692779541,
      "step": 962
    },
    {
      "epoch": 1.26,
      "learning_rate": 4.840116260579557e-05,
      "logits/chosen": -2.3253631591796875,
      "logits/rejected": -2.4139952659606934,
      "logps/chosen": -196.48281860351562,
      "logps/rejected": -258.4675598144531,
      "loss": 0.0903,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2855195999145508,
      "rewards/margins": 4.241164684295654,
      "rewards/rejected": -5.526684284210205,
      "step": 963
    },
    {
      "epoch": 1.26,
      "learning_rate": 4.839749528270345e-05,
      "logits/chosen": -2.1117494106292725,
      "logits/rejected": -2.233508586883545,
      "logps/chosen": -162.23635864257812,
      "logps/rejected": -216.20143127441406,
      "loss": 0.1694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.412933349609375,
      "rewards/margins": 3.0781798362731934,
      "rewards/rejected": -4.49111270904541,
      "step": 964
    },
    {
      "epoch": 1.26,
      "learning_rate": 4.8393823897737634e-05,
      "logits/chosen": -1.8761091232299805,
      "logits/rejected": -1.8636384010314941,
      "logps/chosen": -202.1988525390625,
      "logps/rejected": -255.47442626953125,
      "loss": 0.1544,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0524611473083496,
      "rewards/margins": 3.7740285396575928,
      "rewards/rejected": -4.8264899253845215,
      "step": 965
    },
    {
      "epoch": 1.26,
      "learning_rate": 4.839014845153548e-05,
      "logits/chosen": -2.455805778503418,
      "logits/rejected": -2.4408648014068604,
      "logps/chosen": -265.5133056640625,
      "logps/rejected": -322.55694580078125,
      "loss": 0.1826,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1125422716140747,
      "rewards/margins": 3.8850979804992676,
      "rewards/rejected": -4.997640132904053,
      "step": 966
    },
    {
      "epoch": 1.27,
      "learning_rate": 4.838646894473505e-05,
      "logits/chosen": -1.7058061361312866,
      "logits/rejected": -1.7635576725006104,
      "logps/chosen": -178.57357788085938,
      "logps/rejected": -224.11141967773438,
      "loss": 0.2088,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7308789491653442,
      "rewards/margins": 2.972151279449463,
      "rewards/rejected": -4.703030586242676,
      "step": 967
    },
    {
      "epoch": 1.27,
      "learning_rate": 4.8382785377975116e-05,
      "logits/chosen": -2.187974452972412,
      "logits/rejected": -2.2810630798339844,
      "logps/chosen": -189.5592803955078,
      "logps/rejected": -282.3250732421875,
      "loss": 0.2646,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5101513862609863,
      "rewards/margins": 3.0083746910095215,
      "rewards/rejected": -4.518526077270508,
      "step": 968
    },
    {
      "epoch": 1.27,
      "learning_rate": 4.8379097751895166e-05,
      "logits/chosen": -2.3393805027008057,
      "logits/rejected": -2.3446669578552246,
      "logps/chosen": -280.5021667480469,
      "logps/rejected": -287.1352233886719,
      "loss": 0.1423,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8774653077125549,
      "rewards/margins": 3.9873814582824707,
      "rewards/rejected": -4.864847183227539,
      "step": 969
    },
    {
      "epoch": 1.27,
      "learning_rate": 4.837540606713538e-05,
      "logits/chosen": -2.3770768642425537,
      "logits/rejected": -2.3758389949798584,
      "logps/chosen": -198.17864990234375,
      "logps/rejected": -225.19105529785156,
      "loss": 0.1787,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3526650667190552,
      "rewards/margins": 2.9090161323547363,
      "rewards/rejected": -4.26168155670166,
      "step": 970
    },
    {
      "epoch": 1.27,
      "learning_rate": 4.837171032433663e-05,
      "logits/chosen": -2.226538896560669,
      "logits/rejected": -2.3716554641723633,
      "logps/chosen": -181.35006713867188,
      "logps/rejected": -279.17645263671875,
      "loss": 0.0983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6791074275970459,
      "rewards/margins": 4.5800275802612305,
      "rewards/rejected": -5.259134292602539,
      "step": 971
    },
    {
      "epoch": 1.27,
      "learning_rate": 4.836801052414053e-05,
      "logits/chosen": -1.950883388519287,
      "logits/rejected": -2.080726146697998,
      "logps/chosen": -170.36874389648438,
      "logps/rejected": -215.67941284179688,
      "loss": 0.2373,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.109346866607666,
      "rewards/margins": 2.8225221633911133,
      "rewards/rejected": -3.9318690299987793,
      "step": 972
    },
    {
      "epoch": 1.27,
      "learning_rate": 4.8364306667189364e-05,
      "logits/chosen": -2.23186993598938,
      "logits/rejected": -2.1386542320251465,
      "logps/chosen": -180.2755126953125,
      "logps/rejected": -230.13412475585938,
      "loss": 0.112,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8742038011550903,
      "rewards/margins": 3.692150115966797,
      "rewards/rejected": -4.566353797912598,
      "step": 973
    },
    {
      "epoch": 1.27,
      "learning_rate": 4.836059875412613e-05,
      "logits/chosen": -2.152693748474121,
      "logits/rejected": -2.1664419174194336,
      "logps/chosen": -197.43896484375,
      "logps/rejected": -206.69210815429688,
      "loss": 0.2823,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0961377620697021,
      "rewards/margins": 3.136141538619995,
      "rewards/rejected": -4.2322797775268555,
      "step": 974
    },
    {
      "epoch": 1.28,
      "learning_rate": 4.8356886785594544e-05,
      "logits/chosen": -2.3830044269561768,
      "logits/rejected": -2.392028331756592,
      "logps/chosen": -191.5318603515625,
      "logps/rejected": -222.13458251953125,
      "loss": 0.082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9227029085159302,
      "rewards/margins": 4.220012664794922,
      "rewards/rejected": -5.142715930938721,
      "step": 975
    },
    {
      "epoch": 1.28,
      "learning_rate": 4.835317076223901e-05,
      "logits/chosen": -2.298764228820801,
      "logits/rejected": -2.360914468765259,
      "logps/chosen": -238.4506378173828,
      "logps/rejected": -276.4215087890625,
      "loss": 0.1926,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0809506177902222,
      "rewards/margins": 3.495990037918091,
      "rewards/rejected": -4.576941013336182,
      "step": 976
    },
    {
      "epoch": 1.28,
      "learning_rate": 4.834945068470463e-05,
      "logits/chosen": -2.3851685523986816,
      "logits/rejected": -2.402043342590332,
      "logps/chosen": -215.42071533203125,
      "logps/rejected": -260.2996520996094,
      "loss": 0.1705,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8322378396987915,
      "rewards/margins": 3.276536703109741,
      "rewards/rejected": -4.1087751388549805,
      "step": 977
    },
    {
      "epoch": 1.28,
      "learning_rate": 4.834572655363723e-05,
      "logits/chosen": -2.3521459102630615,
      "logits/rejected": -2.359070301055908,
      "logps/chosen": -172.63397216796875,
      "logps/rejected": -249.02813720703125,
      "loss": 0.1009,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6295977830886841,
      "rewards/margins": 4.736910343170166,
      "rewards/rejected": -5.3665080070495605,
      "step": 978
    },
    {
      "epoch": 1.28,
      "learning_rate": 4.834199836968334e-05,
      "logits/chosen": -2.232576847076416,
      "logits/rejected": -2.2787835597991943,
      "logps/chosen": -212.65316772460938,
      "logps/rejected": -277.7938537597656,
      "loss": 0.1771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9311662912368774,
      "rewards/margins": 4.264695644378662,
      "rewards/rejected": -5.19586181640625,
      "step": 979
    },
    {
      "epoch": 1.28,
      "learning_rate": 4.833826613349016e-05,
      "logits/chosen": -2.2517361640930176,
      "logits/rejected": -2.284072160720825,
      "logps/chosen": -220.1957550048828,
      "logps/rejected": -214.6461181640625,
      "loss": 0.2048,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8580664396286011,
      "rewards/margins": 3.004542589187622,
      "rewards/rejected": -3.8626086711883545,
      "step": 980
    },
    {
      "epoch": 1.28,
      "learning_rate": 4.8334529845705635e-05,
      "logits/chosen": -2.3734512329101562,
      "logits/rejected": -2.3852882385253906,
      "logps/chosen": -185.3246612548828,
      "logps/rejected": -235.97036743164062,
      "loss": 0.0987,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8764241337776184,
      "rewards/margins": 3.519378900527954,
      "rewards/rejected": -4.395802974700928,
      "step": 981
    },
    {
      "epoch": 1.29,
      "learning_rate": 4.833078950697839e-05,
      "logits/chosen": -2.318061113357544,
      "logits/rejected": -2.3952810764312744,
      "logps/chosen": -203.9101104736328,
      "logps/rejected": -264.8881530761719,
      "loss": 0.1093,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5848926305770874,
      "rewards/margins": 3.398397207260132,
      "rewards/rejected": -4.98328971862793,
      "step": 982
    },
    {
      "epoch": 1.29,
      "learning_rate": 4.832704511795776e-05,
      "logits/chosen": -2.257204294204712,
      "logits/rejected": -2.243340015411377,
      "logps/chosen": -220.27609252929688,
      "logps/rejected": -302.30224609375,
      "loss": 0.0464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7187144756317139,
      "rewards/margins": 5.5636396408081055,
      "rewards/rejected": -6.282354354858398,
      "step": 983
    },
    {
      "epoch": 1.29,
      "learning_rate": 4.8323296679293774e-05,
      "logits/chosen": -2.0718610286712646,
      "logits/rejected": -2.163663148880005,
      "logps/chosen": -201.58529663085938,
      "logps/rejected": -284.6639099121094,
      "loss": 0.186,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4365129470825195,
      "rewards/margins": 3.119493007659912,
      "rewards/rejected": -4.55600643157959,
      "step": 984
    },
    {
      "epoch": 1.29,
      "learning_rate": 4.831954419163719e-05,
      "logits/chosen": -2.378993034362793,
      "logits/rejected": -2.4145779609680176,
      "logps/chosen": -194.3405303955078,
      "logps/rejected": -258.2578430175781,
      "loss": 0.1569,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0772855281829834,
      "rewards/margins": 3.5224766731262207,
      "rewards/rejected": -4.599761962890625,
      "step": 985
    },
    {
      "epoch": 1.29,
      "learning_rate": 4.8315787655639436e-05,
      "logits/chosen": -2.158259391784668,
      "logits/rejected": -2.242321729660034,
      "logps/chosen": -172.10662841796875,
      "logps/rejected": -238.54212951660156,
      "loss": 0.1116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0470584630966187,
      "rewards/margins": 3.594759225845337,
      "rewards/rejected": -4.641817569732666,
      "step": 986
    },
    {
      "epoch": 1.29,
      "learning_rate": 4.8312027071952645e-05,
      "logits/chosen": -2.3166654109954834,
      "logits/rejected": -2.371119499206543,
      "logps/chosen": -201.59706115722656,
      "logps/rejected": -208.94171142578125,
      "loss": 0.1019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.161280632019043,
      "rewards/margins": 3.300464630126953,
      "rewards/rejected": -4.461745262145996,
      "step": 987
    },
    {
      "epoch": 1.29,
      "learning_rate": 4.8308262441229704e-05,
      "logits/chosen": -2.368316888809204,
      "logits/rejected": -2.464874029159546,
      "logps/chosen": -227.5435333251953,
      "logps/rejected": -269.635009765625,
      "loss": 0.1186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2794337272644043,
      "rewards/margins": 3.7162654399871826,
      "rewards/rejected": -4.995699882507324,
      "step": 988
    },
    {
      "epoch": 1.29,
      "learning_rate": 4.830449376412413e-05,
      "logits/chosen": -2.4263885021209717,
      "logits/rejected": -2.330914258956909,
      "logps/chosen": -174.01873779296875,
      "logps/rejected": -176.23687744140625,
      "loss": 0.2376,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1324949264526367,
      "rewards/margins": 2.7193217277526855,
      "rewards/rejected": -3.8518168926239014,
      "step": 989
    },
    {
      "epoch": 1.3,
      "learning_rate": 4.8300721041290194e-05,
      "logits/chosen": -2.482351303100586,
      "logits/rejected": -2.511679172515869,
      "logps/chosen": -258.9258117675781,
      "logps/rejected": -289.4918212890625,
      "loss": 0.0891,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4783955812454224,
      "rewards/margins": 3.5242300033569336,
      "rewards/rejected": -5.002625465393066,
      "step": 990
    },
    {
      "epoch": 1.3,
      "learning_rate": 4.829694427338284e-05,
      "logits/chosen": -2.210787773132324,
      "logits/rejected": -2.2695932388305664,
      "logps/chosen": -242.7936553955078,
      "logps/rejected": -285.85845947265625,
      "loss": 0.0737,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1135588884353638,
      "rewards/margins": 4.644796371459961,
      "rewards/rejected": -5.758354663848877,
      "step": 991
    },
    {
      "epoch": 1.3,
      "learning_rate": 4.829316346105773e-05,
      "logits/chosen": -2.5152206420898438,
      "logits/rejected": -2.598931312561035,
      "logps/chosen": -233.4760284423828,
      "logps/rejected": -257.0856628417969,
      "loss": 0.1722,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4175981283187866,
      "rewards/margins": 3.751023530960083,
      "rewards/rejected": -5.168622016906738,
      "step": 992
    },
    {
      "epoch": 1.3,
      "learning_rate": 4.828937860497124e-05,
      "logits/chosen": -1.9849010705947876,
      "logits/rejected": -2.02589750289917,
      "logps/chosen": -202.5492706298828,
      "logps/rejected": -240.67250061035156,
      "loss": 0.2435,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1498560905456543,
      "rewards/margins": 2.9630770683288574,
      "rewards/rejected": -4.112933158874512,
      "step": 993
    },
    {
      "epoch": 1.3,
      "learning_rate": 4.82855897057804e-05,
      "logits/chosen": -2.2394447326660156,
      "logits/rejected": -2.2644007205963135,
      "logps/chosen": -200.177978515625,
      "logps/rejected": -232.31613159179688,
      "loss": 0.1656,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2112644910812378,
      "rewards/margins": 3.5165750980377197,
      "rewards/rejected": -4.727839469909668,
      "step": 994
    },
    {
      "epoch": 1.3,
      "learning_rate": 4.8281796764143e-05,
      "logits/chosen": -2.1232497692108154,
      "logits/rejected": -2.1741788387298584,
      "logps/chosen": -187.11965942382812,
      "logps/rejected": -211.57391357421875,
      "loss": 0.0946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9103628396987915,
      "rewards/margins": 3.7156426906585693,
      "rewards/rejected": -4.62600564956665,
      "step": 995
    },
    {
      "epoch": 1.3,
      "learning_rate": 4.8277999780717496e-05,
      "logits/chosen": -2.3636045455932617,
      "logits/rejected": -2.3993918895721436,
      "logps/chosen": -209.7058868408203,
      "logps/rejected": -273.6556396484375,
      "loss": 0.1253,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.197162389755249,
      "rewards/margins": 4.630073547363281,
      "rewards/rejected": -5.827235698699951,
      "step": 996
    },
    {
      "epoch": 1.3,
      "learning_rate": 4.827419875616306e-05,
      "logits/chosen": -2.4287328720092773,
      "logits/rejected": -2.444946765899658,
      "logps/chosen": -187.06781005859375,
      "logps/rejected": -227.0997314453125,
      "loss": 0.3696,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.5417091846466064,
      "rewards/margins": 3.166154623031616,
      "rewards/rejected": -4.707863807678223,
      "step": 997
    },
    {
      "epoch": 1.31,
      "learning_rate": 4.827039369113956e-05,
      "logits/chosen": -2.3056039810180664,
      "logits/rejected": -2.3643782138824463,
      "logps/chosen": -218.290283203125,
      "logps/rejected": -289.4818420410156,
      "loss": 0.0622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1654748916625977,
      "rewards/margins": 4.145327091217041,
      "rewards/rejected": -5.310801982879639,
      "step": 998
    },
    {
      "epoch": 1.31,
      "learning_rate": 4.826658458630756e-05,
      "logits/chosen": -2.3559858798980713,
      "logits/rejected": -2.4298417568206787,
      "logps/chosen": -279.0400695800781,
      "logps/rejected": -276.64044189453125,
      "loss": 0.1911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6909762620925903,
      "rewards/margins": 3.3049442768096924,
      "rewards/rejected": -4.995920658111572,
      "step": 999
    },
    {
      "epoch": 1.31,
      "learning_rate": 4.826277144232834e-05,
      "logits/chosen": -2.3466007709503174,
      "logits/rejected": -2.376455068588257,
      "logps/chosen": -215.84930419921875,
      "logps/rejected": -249.32769775390625,
      "loss": 0.1875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2421891689300537,
      "rewards/margins": 4.225651264190674,
      "rewards/rejected": -5.467840194702148,
      "step": 1000
    },
    {
      "epoch": 1.31,
      "learning_rate": 4.825895425986386e-05,
      "logits/chosen": -2.164769172668457,
      "logits/rejected": -2.1837494373321533,
      "logps/chosen": -250.56695556640625,
      "logps/rejected": -270.6837463378906,
      "loss": 0.1681,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4974141120910645,
      "rewards/margins": 3.5974225997924805,
      "rewards/rejected": -5.094837188720703,
      "step": 1001
    },
    {
      "epoch": 1.31,
      "learning_rate": 4.82551330395768e-05,
      "logits/chosen": -2.248732328414917,
      "logits/rejected": -2.308931589126587,
      "logps/chosen": -226.01544189453125,
      "logps/rejected": -276.2228698730469,
      "loss": 0.1258,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.646399736404419,
      "rewards/margins": 3.5327749252319336,
      "rewards/rejected": -5.179174423217773,
      "step": 1002
    },
    {
      "epoch": 1.31,
      "learning_rate": 4.825130778213055e-05,
      "logits/chosen": -1.9366941452026367,
      "logits/rejected": -1.9974042177200317,
      "logps/chosen": -248.90155029296875,
      "logps/rejected": -284.5340576171875,
      "loss": 0.1246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5386114120483398,
      "rewards/margins": 4.703797340393066,
      "rewards/rejected": -6.242408752441406,
      "step": 1003
    },
    {
      "epoch": 1.31,
      "learning_rate": 4.8247478488189176e-05,
      "logits/chosen": -2.381505012512207,
      "logits/rejected": -2.374134063720703,
      "logps/chosen": -215.34832763671875,
      "logps/rejected": -241.09263610839844,
      "loss": 0.1462,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2310172319412231,
      "rewards/margins": 3.3694674968719482,
      "rewards/rejected": -4.600484848022461,
      "step": 1004
    },
    {
      "epoch": 1.32,
      "learning_rate": 4.824364515841745e-05,
      "logits/chosen": -2.39587664604187,
      "logits/rejected": -2.445971727371216,
      "logps/chosen": -199.75390625,
      "logps/rejected": -250.50318908691406,
      "loss": 0.1496,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2829456329345703,
      "rewards/margins": 3.5297467708587646,
      "rewards/rejected": -4.812692642211914,
      "step": 1005
    },
    {
      "epoch": 1.32,
      "learning_rate": 4.8239807793480845e-05,
      "logits/chosen": -2.064811944961548,
      "logits/rejected": -2.2144877910614014,
      "logps/chosen": -206.80836486816406,
      "logps/rejected": -274.880859375,
      "loss": 0.0629,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6409518718719482,
      "rewards/margins": 4.403507709503174,
      "rewards/rejected": -6.044458866119385,
      "step": 1006
    },
    {
      "epoch": 1.32,
      "learning_rate": 4.823596639404555e-05,
      "logits/chosen": -1.9515126943588257,
      "logits/rejected": -1.9221199750900269,
      "logps/chosen": -183.7147216796875,
      "logps/rejected": -199.12330627441406,
      "loss": 0.2176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4174538850784302,
      "rewards/margins": 2.61428165435791,
      "rewards/rejected": -4.031735420227051,
      "step": 1007
    },
    {
      "epoch": 1.32,
      "learning_rate": 4.8232120960778444e-05,
      "logits/chosen": -2.1660711765289307,
      "logits/rejected": -2.1986541748046875,
      "logps/chosen": -219.20086669921875,
      "logps/rejected": -308.8476867675781,
      "loss": 0.1746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5317251682281494,
      "rewards/margins": 4.233796119689941,
      "rewards/rejected": -5.765522003173828,
      "step": 1008
    },
    {
      "epoch": 1.32,
      "learning_rate": 4.822827149434709e-05,
      "logits/chosen": -2.3264455795288086,
      "logits/rejected": -2.340402126312256,
      "logps/chosen": -206.29624938964844,
      "logps/rejected": -227.3938446044922,
      "loss": 0.2069,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2022267580032349,
      "rewards/margins": 3.670454263687134,
      "rewards/rejected": -4.872681140899658,
      "step": 1009
    },
    {
      "epoch": 1.32,
      "learning_rate": 4.822441799541979e-05,
      "logits/chosen": -1.881190538406372,
      "logits/rejected": -1.9560871124267578,
      "logps/chosen": -176.16571044921875,
      "logps/rejected": -228.5875244140625,
      "loss": 0.3731,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.7666243314743042,
      "rewards/margins": 2.6788949966430664,
      "rewards/rejected": -4.44551944732666,
      "step": 1010
    },
    {
      "epoch": 1.32,
      "learning_rate": 4.82205604646655e-05,
      "logits/chosen": -1.936165690422058,
      "logits/rejected": -1.9999370574951172,
      "logps/chosen": -175.9329833984375,
      "logps/rejected": -251.57110595703125,
      "loss": 0.1323,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2743474245071411,
      "rewards/margins": 4.712624549865723,
      "rewards/rejected": -5.986972332000732,
      "step": 1011
    },
    {
      "epoch": 1.32,
      "learning_rate": 4.8216698902753915e-05,
      "logits/chosen": -2.4014575481414795,
      "logits/rejected": -2.3882806301116943,
      "logps/chosen": -235.85289001464844,
      "logps/rejected": -276.8750915527344,
      "loss": 0.1432,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5504268407821655,
      "rewards/margins": 4.225334644317627,
      "rewards/rejected": -5.775761127471924,
      "step": 1012
    },
    {
      "epoch": 1.33,
      "learning_rate": 4.8212833310355397e-05,
      "logits/chosen": -2.2567152976989746,
      "logits/rejected": -2.307617425918579,
      "logps/chosen": -250.41036987304688,
      "logps/rejected": -298.62841796875,
      "loss": 0.1228,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.641356110572815,
      "rewards/margins": 3.9862754344940186,
      "rewards/rejected": -5.627632141113281,
      "step": 1013
    },
    {
      "epoch": 1.33,
      "learning_rate": 4.820896368814104e-05,
      "logits/chosen": -2.222958564758301,
      "logits/rejected": -2.243933916091919,
      "logps/chosen": -144.96896362304688,
      "logps/rejected": -212.0235595703125,
      "loss": 0.115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9734640121459961,
      "rewards/margins": 3.973369598388672,
      "rewards/rejected": -4.946833610534668,
      "step": 1014
    },
    {
      "epoch": 1.33,
      "learning_rate": 4.8205090036782626e-05,
      "logits/chosen": -2.0626208782196045,
      "logits/rejected": -2.0410847663879395,
      "logps/chosen": -171.70816040039062,
      "logps/rejected": -231.00027465820312,
      "loss": 0.1491,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3891879320144653,
      "rewards/margins": 3.029548406600952,
      "rewards/rejected": -4.418736457824707,
      "step": 1015
    },
    {
      "epoch": 1.33,
      "learning_rate": 4.820121235695261e-05,
      "logits/chosen": -2.0706686973571777,
      "logits/rejected": -2.0150628089904785,
      "logps/chosen": -160.81874084472656,
      "logps/rejected": -187.8358917236328,
      "loss": 0.2422,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6058461666107178,
      "rewards/margins": 2.6923718452453613,
      "rewards/rejected": -4.298218250274658,
      "step": 1016
    },
    {
      "epoch": 1.33,
      "learning_rate": 4.8197330649324184e-05,
      "logits/chosen": -2.411139965057373,
      "logits/rejected": -2.2218780517578125,
      "logps/chosen": -143.00469970703125,
      "logps/rejected": -175.34141540527344,
      "loss": 0.2557,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3263788223266602,
      "rewards/margins": 3.335325241088867,
      "rewards/rejected": -4.661704063415527,
      "step": 1017
    },
    {
      "epoch": 1.33,
      "learning_rate": 4.819344491457122e-05,
      "logits/chosen": -2.058603048324585,
      "logits/rejected": -2.1647167205810547,
      "logps/chosen": -168.77676391601562,
      "logps/rejected": -233.14918518066406,
      "loss": 0.1883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0997314453125,
      "rewards/margins": 3.1378016471862793,
      "rewards/rejected": -5.2375335693359375,
      "step": 1018
    },
    {
      "epoch": 1.33,
      "learning_rate": 4.818955515336829e-05,
      "logits/chosen": -2.3335368633270264,
      "logits/rejected": -2.343855381011963,
      "logps/chosen": -220.887939453125,
      "logps/rejected": -275.09100341796875,
      "loss": 0.2941,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5219594240188599,
      "rewards/margins": 4.077702045440674,
      "rewards/rejected": -5.599661350250244,
      "step": 1019
    },
    {
      "epoch": 1.33,
      "learning_rate": 4.8185661366390676e-05,
      "logits/chosen": -2.0001189708709717,
      "logits/rejected": -2.10128116607666,
      "logps/chosen": -237.78350830078125,
      "logps/rejected": -302.6692810058594,
      "loss": 0.2217,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.2568397521972656,
      "rewards/margins": 3.168138027191162,
      "rewards/rejected": -6.4249773025512695,
      "step": 1020
    },
    {
      "epoch": 1.34,
      "learning_rate": 4.8181763554314345e-05,
      "logits/chosen": -2.1002883911132812,
      "logits/rejected": -2.1318533420562744,
      "logps/chosen": -215.0624237060547,
      "logps/rejected": -272.5454406738281,
      "loss": 0.0886,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1878482103347778,
      "rewards/margins": 5.048128128051758,
      "rewards/rejected": -6.2359771728515625,
      "step": 1021
    },
    {
      "epoch": 1.34,
      "learning_rate": 4.8177861717815976e-05,
      "logits/chosen": -2.43477463722229,
      "logits/rejected": -2.367626905441284,
      "logps/chosen": -170.18011474609375,
      "logps/rejected": -194.78160095214844,
      "loss": 0.2638,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.694594383239746,
      "rewards/margins": 2.889676094055176,
      "rewards/rejected": -4.584270000457764,
      "step": 1022
    },
    {
      "epoch": 1.34,
      "learning_rate": 4.8173955857572924e-05,
      "logits/chosen": -2.2585108280181885,
      "logits/rejected": -2.2841644287109375,
      "logps/chosen": -221.76345825195312,
      "logps/rejected": -282.0390930175781,
      "loss": 0.123,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.552736759185791,
      "rewards/margins": 4.385633945465088,
      "rewards/rejected": -5.938370704650879,
      "step": 1023
    },
    {
      "epoch": 1.34,
      "learning_rate": 4.817004597426327e-05,
      "logits/chosen": -2.252422332763672,
      "logits/rejected": -2.369584083557129,
      "logps/chosen": -198.19515991210938,
      "logps/rejected": -256.17254638671875,
      "loss": 0.0797,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3669312000274658,
      "rewards/margins": 4.838632583618164,
      "rewards/rejected": -6.205563545227051,
      "step": 1024
    },
    {
      "epoch": 1.34,
      "learning_rate": 4.816613206856577e-05,
      "logits/chosen": -1.7100903987884521,
      "logits/rejected": -1.7708444595336914,
      "logps/chosen": -224.86050415039062,
      "logps/rejected": -296.8453674316406,
      "loss": 0.1638,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.458321452140808,
      "rewards/margins": 4.833914756774902,
      "rewards/rejected": -6.292235851287842,
      "step": 1025
    },
    {
      "epoch": 1.34,
      "learning_rate": 4.8162214141159914e-05,
      "logits/chosen": -2.167717933654785,
      "logits/rejected": -2.2409541606903076,
      "logps/chosen": -137.0130615234375,
      "logps/rejected": -195.80404663085938,
      "loss": 0.2602,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.234320878982544,
      "rewards/margins": 3.5150794982910156,
      "rewards/rejected": -4.749400615692139,
      "step": 1026
    },
    {
      "epoch": 1.34,
      "learning_rate": 4.815829219272584e-05,
      "logits/chosen": -2.1251039505004883,
      "logits/rejected": -2.1560146808624268,
      "logps/chosen": -217.72801208496094,
      "logps/rejected": -259.56439208984375,
      "loss": 0.1545,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7046087980270386,
      "rewards/margins": 3.3975937366485596,
      "rewards/rejected": -5.102202415466309,
      "step": 1027
    },
    {
      "epoch": 1.35,
      "learning_rate": 4.815436622394441e-05,
      "logits/chosen": -2.1002719402313232,
      "logits/rejected": -2.108283042907715,
      "logps/chosen": -123.21249389648438,
      "logps/rejected": -177.59942626953125,
      "loss": 0.334,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.118762493133545,
      "rewards/margins": 2.3827548027038574,
      "rewards/rejected": -3.5015172958374023,
      "step": 1028
    },
    {
      "epoch": 1.35,
      "learning_rate": 4.815043623549721e-05,
      "logits/chosen": -1.9848045110702515,
      "logits/rejected": -2.0297982692718506,
      "logps/chosen": -191.30886840820312,
      "logps/rejected": -238.38487243652344,
      "loss": 0.1696,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3014191389083862,
      "rewards/margins": 3.1689629554748535,
      "rewards/rejected": -4.470382213592529,
      "step": 1029
    },
    {
      "epoch": 1.35,
      "learning_rate": 4.814650222806647e-05,
      "logits/chosen": -2.1499392986297607,
      "logits/rejected": -2.2093987464904785,
      "logps/chosen": -200.4322509765625,
      "logps/rejected": -266.98614501953125,
      "loss": 0.1951,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3251994848251343,
      "rewards/margins": 3.7016544342041016,
      "rewards/rejected": -5.026854038238525,
      "step": 1030
    },
    {
      "epoch": 1.35,
      "learning_rate": 4.8142564202335155e-05,
      "logits/chosen": -2.294783115386963,
      "logits/rejected": -2.1916940212249756,
      "logps/chosen": -168.14263916015625,
      "logps/rejected": -178.0428466796875,
      "loss": 0.2519,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3096660375595093,
      "rewards/margins": 2.79194974899292,
      "rewards/rejected": -4.101615905761719,
      "step": 1031
    },
    {
      "epoch": 1.35,
      "learning_rate": 4.813862215898692e-05,
      "logits/chosen": -2.161398410797119,
      "logits/rejected": -2.232086181640625,
      "logps/chosen": -197.67620849609375,
      "logps/rejected": -290.12420654296875,
      "loss": 0.1613,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.239977478981018,
      "rewards/margins": 3.1511924266815186,
      "rewards/rejected": -4.391169548034668,
      "step": 1032
    },
    {
      "epoch": 1.35,
      "learning_rate": 4.813467609870611e-05,
      "logits/chosen": -2.2221438884735107,
      "logits/rejected": -2.1995956897735596,
      "logps/chosen": -224.52113342285156,
      "logps/rejected": -237.9773712158203,
      "loss": 0.0758,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0239717960357666,
      "rewards/margins": 4.245767116546631,
      "rewards/rejected": -5.269738674163818,
      "step": 1033
    },
    {
      "epoch": 1.35,
      "learning_rate": 4.813072602217778e-05,
      "logits/chosen": -2.0670764446258545,
      "logits/rejected": -2.113513469696045,
      "logps/chosen": -233.91201782226562,
      "logps/rejected": -259.1944885253906,
      "loss": 0.2214,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0860904455184937,
      "rewards/margins": 4.289690971374512,
      "rewards/rejected": -5.375781059265137,
      "step": 1034
    },
    {
      "epoch": 1.35,
      "learning_rate": 4.8126771930087674e-05,
      "logits/chosen": -2.3435795307159424,
      "logits/rejected": -2.508849620819092,
      "logps/chosen": -174.04713439941406,
      "logps/rejected": -249.47862243652344,
      "loss": 0.2356,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.164699673652649,
      "rewards/margins": 4.200926303863525,
      "rewards/rejected": -5.365626335144043,
      "step": 1035
    },
    {
      "epoch": 1.36,
      "learning_rate": 4.8122813823122225e-05,
      "logits/chosen": -2.4261474609375,
      "logits/rejected": -2.459974527359009,
      "logps/chosen": -207.60198974609375,
      "logps/rejected": -263.3089294433594,
      "loss": 0.1271,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1393440961837769,
      "rewards/margins": 4.021913528442383,
      "rewards/rejected": -5.161257266998291,
      "step": 1036
    },
    {
      "epoch": 1.36,
      "learning_rate": 4.8118851701968584e-05,
      "logits/chosen": -2.2980728149414062,
      "logits/rejected": -2.3554952144622803,
      "logps/chosen": -187.2412872314453,
      "logps/rejected": -224.6619873046875,
      "loss": 0.126,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9625990390777588,
      "rewards/margins": 3.2137532234191895,
      "rewards/rejected": -4.176352500915527,
      "step": 1037
    },
    {
      "epoch": 1.36,
      "learning_rate": 4.811488556731457e-05,
      "logits/chosen": -1.995209813117981,
      "logits/rejected": -2.0629470348358154,
      "logps/chosen": -194.26153564453125,
      "logps/rejected": -252.84649658203125,
      "loss": 0.0712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7154987454414368,
      "rewards/margins": 4.5765485763549805,
      "rewards/rejected": -5.292047023773193,
      "step": 1038
    },
    {
      "epoch": 1.36,
      "learning_rate": 4.8110915419848734e-05,
      "logits/chosen": -2.278034210205078,
      "logits/rejected": -2.3754935264587402,
      "logps/chosen": -180.0498809814453,
      "logps/rejected": -240.5858154296875,
      "loss": 0.1379,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1113700866699219,
      "rewards/margins": 3.6900339126586914,
      "rewards/rejected": -4.801403999328613,
      "step": 1039
    },
    {
      "epoch": 1.36,
      "learning_rate": 4.8106941260260296e-05,
      "logits/chosen": -2.460465908050537,
      "logits/rejected": -2.4936022758483887,
      "logps/chosen": -239.6426544189453,
      "logps/rejected": -292.16949462890625,
      "loss": 0.0555,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9402986764907837,
      "rewards/margins": 4.606878280639648,
      "rewards/rejected": -5.547176837921143,
      "step": 1040
    },
    {
      "epoch": 1.36,
      "learning_rate": 4.8102963089239185e-05,
      "logits/chosen": -2.076064348220825,
      "logits/rejected": -2.1251323223114014,
      "logps/chosen": -140.34951782226562,
      "logps/rejected": -206.3195343017578,
      "loss": 0.3204,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3504629135131836,
      "rewards/margins": 3.5889742374420166,
      "rewards/rejected": -4.939437389373779,
      "step": 1041
    },
    {
      "epoch": 1.36,
      "learning_rate": 4.809898090747601e-05,
      "logits/chosen": -2.289942502975464,
      "logits/rejected": -2.3287506103515625,
      "logps/chosen": -213.01470947265625,
      "logps/rejected": -265.6434020996094,
      "loss": 0.1479,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9047419428825378,
      "rewards/margins": 3.5847079753875732,
      "rewards/rejected": -4.489449977874756,
      "step": 1042
    },
    {
      "epoch": 1.36,
      "learning_rate": 4.809499471566211e-05,
      "logits/chosen": -2.0999417304992676,
      "logits/rejected": -2.199672222137451,
      "logps/chosen": -178.01663208007812,
      "logps/rejected": -248.11026000976562,
      "loss": 0.105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8936255574226379,
      "rewards/margins": 4.213724613189697,
      "rewards/rejected": -5.107350826263428,
      "step": 1043
    },
    {
      "epoch": 1.37,
      "learning_rate": 4.809100451448949e-05,
      "logits/chosen": -2.2089450359344482,
      "logits/rejected": -2.1612417697906494,
      "logps/chosen": -157.15957641601562,
      "logps/rejected": -182.26841735839844,
      "loss": 0.2212,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1464567184448242,
      "rewards/margins": 3.457570791244507,
      "rewards/rejected": -4.604027271270752,
      "step": 1044
    },
    {
      "epoch": 1.37,
      "learning_rate": 4.8087010304650866e-05,
      "logits/chosen": -2.1104516983032227,
      "logits/rejected": -2.153797149658203,
      "logps/chosen": -222.3661651611328,
      "logps/rejected": -295.9107666015625,
      "loss": 0.0823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4407971203327179,
      "rewards/margins": 4.490683078765869,
      "rewards/rejected": -4.9314799308776855,
      "step": 1045
    },
    {
      "epoch": 1.37,
      "learning_rate": 4.808301208683963e-05,
      "logits/chosen": -2.568639039993286,
      "logits/rejected": -2.5642662048339844,
      "logps/chosen": -239.6685333251953,
      "logps/rejected": -248.938720703125,
      "loss": 0.1162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3235012292861938,
      "rewards/margins": 4.261366844177246,
      "rewards/rejected": -5.58486795425415,
      "step": 1046
    },
    {
      "epoch": 1.37,
      "learning_rate": 4.8079009861749904e-05,
      "logits/chosen": -2.3855719566345215,
      "logits/rejected": -2.369443893432617,
      "logps/chosen": -236.78912353515625,
      "logps/rejected": -273.231689453125,
      "loss": 0.0566,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4863171577453613,
      "rewards/margins": 4.7871551513671875,
      "rewards/rejected": -6.273472785949707,
      "step": 1047
    },
    {
      "epoch": 1.37,
      "learning_rate": 4.807500363007647e-05,
      "logits/chosen": -2.229086399078369,
      "logits/rejected": -2.2423181533813477,
      "logps/chosen": -165.6907958984375,
      "logps/rejected": -186.762451171875,
      "loss": 0.1962,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7180208563804626,
      "rewards/margins": 2.7720799446105957,
      "rewards/rejected": -3.490100860595703,
      "step": 1048
    },
    {
      "epoch": 1.37,
      "learning_rate": 4.8070993392514826e-05,
      "logits/chosen": -2.305532693862915,
      "logits/rejected": -2.322361469268799,
      "logps/chosen": -202.0149688720703,
      "logps/rejected": -243.12847900390625,
      "loss": 0.1414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1048834323883057,
      "rewards/margins": 4.179681301116943,
      "rewards/rejected": -5.284564971923828,
      "step": 1049
    },
    {
      "epoch": 1.37,
      "learning_rate": 4.806697914976116e-05,
      "logits/chosen": -2.4315245151519775,
      "logits/rejected": -2.420684337615967,
      "logps/chosen": -206.34152221679688,
      "logps/rejected": -244.23727416992188,
      "loss": 0.171,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3189445734024048,
      "rewards/margins": 3.540530204772949,
      "rewards/rejected": -4.859475135803223,
      "step": 1050
    },
    {
      "epoch": 1.38,
      "learning_rate": 4.806296090251236e-05,
      "logits/chosen": -2.0065677165985107,
      "logits/rejected": -2.1579909324645996,
      "logps/chosen": -167.84841918945312,
      "logps/rejected": -246.53677368164062,
      "loss": 0.1576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1663308143615723,
      "rewards/margins": 3.5337471961975098,
      "rewards/rejected": -4.70007848739624,
      "step": 1051
    },
    {
      "epoch": 1.38,
      "learning_rate": 4.805893865146601e-05,
      "logits/chosen": -2.1749401092529297,
      "logits/rejected": -2.2115371227264404,
      "logps/chosen": -235.0357208251953,
      "logps/rejected": -288.3175964355469,
      "loss": 0.1302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5622204542160034,
      "rewards/margins": 3.5101852416992188,
      "rewards/rejected": -5.0724053382873535,
      "step": 1052
    },
    {
      "epoch": 1.38,
      "learning_rate": 4.805491239732037e-05,
      "logits/chosen": -2.3291127681732178,
      "logits/rejected": -2.414362668991089,
      "logps/chosen": -173.30560302734375,
      "logps/rejected": -301.0763854980469,
      "loss": 0.0217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7221314907073975,
      "rewards/margins": 6.185711860656738,
      "rewards/rejected": -7.907843589782715,
      "step": 1053
    },
    {
      "epoch": 1.38,
      "learning_rate": 4.8050882140774425e-05,
      "logits/chosen": -2.316331624984741,
      "logits/rejected": -2.383364677429199,
      "logps/chosen": -177.23760986328125,
      "logps/rejected": -202.95751953125,
      "loss": 0.2219,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.775866985321045,
      "rewards/margins": 2.098893404006958,
      "rewards/rejected": -3.874760150909424,
      "step": 1054
    },
    {
      "epoch": 1.38,
      "learning_rate": 4.8046847882527826e-05,
      "logits/chosen": -2.2674598693847656,
      "logits/rejected": -2.2072408199310303,
      "logps/chosen": -173.46591186523438,
      "logps/rejected": -221.97886657714844,
      "loss": 0.1288,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.116098403930664,
      "rewards/margins": 3.485488176345825,
      "rewards/rejected": -4.60158634185791,
      "step": 1055
    },
    {
      "epoch": 1.38,
      "learning_rate": 4.8042809623280946e-05,
      "logits/chosen": -2.2563393115997314,
      "logits/rejected": -2.3027093410491943,
      "logps/chosen": -178.65032958984375,
      "logps/rejected": -210.56443786621094,
      "loss": 0.2929,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6832330226898193,
      "rewards/margins": 3.0855979919433594,
      "rewards/rejected": -4.768831253051758,
      "step": 1056
    },
    {
      "epoch": 1.38,
      "learning_rate": 4.803876736373483e-05,
      "logits/chosen": -2.36661958694458,
      "logits/rejected": -2.373730421066284,
      "logps/chosen": -216.2982940673828,
      "logps/rejected": -270.1596374511719,
      "loss": 0.2312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5078752040863037,
      "rewards/margins": 3.428542137145996,
      "rewards/rejected": -4.936417579650879,
      "step": 1057
    },
    {
      "epoch": 1.38,
      "learning_rate": 4.8034721104591215e-05,
      "logits/chosen": -2.025160074234009,
      "logits/rejected": -2.0265681743621826,
      "logps/chosen": -213.1846160888672,
      "logps/rejected": -244.3919677734375,
      "loss": 0.3887,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.756600022315979,
      "rewards/margins": 3.3049325942993164,
      "rewards/rejected": -5.061532497406006,
      "step": 1058
    },
    {
      "epoch": 1.39,
      "learning_rate": 4.803067084655257e-05,
      "logits/chosen": -2.212038278579712,
      "logits/rejected": -2.2199130058288574,
      "logps/chosen": -200.26730346679688,
      "logps/rejected": -226.02320861816406,
      "loss": 0.1888,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1899981498718262,
      "rewards/margins": 3.551909923553467,
      "rewards/rejected": -4.741908073425293,
      "step": 1059
    },
    {
      "epoch": 1.39,
      "learning_rate": 4.802661659032202e-05,
      "logits/chosen": -2.1756439208984375,
      "logits/rejected": -2.2646827697753906,
      "logps/chosen": -170.65829467773438,
      "logps/rejected": -227.00344848632812,
      "loss": 0.2532,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4983477592468262,
      "rewards/margins": 2.899458408355713,
      "rewards/rejected": -4.397806167602539,
      "step": 1060
    },
    {
      "epoch": 1.39,
      "learning_rate": 4.802255833660338e-05,
      "logits/chosen": -2.0890917778015137,
      "logits/rejected": -2.20941424369812,
      "logps/chosen": -194.01486206054688,
      "logps/rejected": -247.7682342529297,
      "loss": 0.1442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1267577409744263,
      "rewards/margins": 4.959884166717529,
      "rewards/rejected": -6.086641788482666,
      "step": 1061
    },
    {
      "epoch": 1.39,
      "learning_rate": 4.8018496086101194e-05,
      "logits/chosen": -2.4077579975128174,
      "logits/rejected": -2.5162241458892822,
      "logps/chosen": -242.92034912109375,
      "logps/rejected": -266.2312316894531,
      "loss": 0.1722,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1819337606430054,
      "rewards/margins": 3.8653903007507324,
      "rewards/rejected": -5.047324180603027,
      "step": 1062
    },
    {
      "epoch": 1.39,
      "learning_rate": 4.801442983952067e-05,
      "logits/chosen": -2.206264019012451,
      "logits/rejected": -2.329880952835083,
      "logps/chosen": -178.778076171875,
      "logps/rejected": -243.94203186035156,
      "loss": 0.2205,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6328147649765015,
      "rewards/margins": 3.6644177436828613,
      "rewards/rejected": -5.297232627868652,
      "step": 1063
    },
    {
      "epoch": 1.39,
      "learning_rate": 4.8010359597567736e-05,
      "logits/chosen": -2.256436347961426,
      "logits/rejected": -2.3086161613464355,
      "logps/chosen": -194.04498291015625,
      "logps/rejected": -268.311279296875,
      "loss": 0.1495,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.80282062292099,
      "rewards/margins": 4.107694625854492,
      "rewards/rejected": -4.910515308380127,
      "step": 1064
    },
    {
      "epoch": 1.39,
      "learning_rate": 4.8006285360948976e-05,
      "logits/chosen": -2.2415833473205566,
      "logits/rejected": -2.3424930572509766,
      "logps/chosen": -214.00341796875,
      "logps/rejected": -264.5972900390625,
      "loss": 0.1117,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2024343013763428,
      "rewards/margins": 3.891465425491333,
      "rewards/rejected": -5.093900203704834,
      "step": 1065
    },
    {
      "epoch": 1.4,
      "learning_rate": 4.8002207130371705e-05,
      "logits/chosen": -2.282310724258423,
      "logits/rejected": -2.3069472312927246,
      "logps/chosen": -175.22569274902344,
      "logps/rejected": -244.1259307861328,
      "loss": 0.1882,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2420991659164429,
      "rewards/margins": 3.709995985031128,
      "rewards/rejected": -4.9520955085754395,
      "step": 1066
    },
    {
      "epoch": 1.4,
      "learning_rate": 4.7998124906543906e-05,
      "logits/chosen": -2.2816646099090576,
      "logits/rejected": -2.3789501190185547,
      "logps/chosen": -186.17849731445312,
      "logps/rejected": -252.48399353027344,
      "loss": 0.226,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2462586164474487,
      "rewards/margins": 3.481846332550049,
      "rewards/rejected": -4.728104591369629,
      "step": 1067
    },
    {
      "epoch": 1.4,
      "learning_rate": 4.799403869017427e-05,
      "logits/chosen": -2.060042142868042,
      "logits/rejected": -2.231752395629883,
      "logps/chosen": -158.0439910888672,
      "logps/rejected": -194.29510498046875,
      "loss": 0.1857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7000864744186401,
      "rewards/margins": 3.765867233276367,
      "rewards/rejected": -4.465953826904297,
      "step": 1068
    },
    {
      "epoch": 1.4,
      "learning_rate": 4.798994848197218e-05,
      "logits/chosen": -2.6301236152648926,
      "logits/rejected": -2.6716156005859375,
      "logps/chosen": -189.2099151611328,
      "logps/rejected": -236.8180694580078,
      "loss": 0.3166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7804349660873413,
      "rewards/margins": 2.868950366973877,
      "rewards/rejected": -4.649385929107666,
      "step": 1069
    },
    {
      "epoch": 1.4,
      "learning_rate": 4.79858542826477e-05,
      "logits/chosen": -2.251466751098633,
      "logits/rejected": -2.363938808441162,
      "logps/chosen": -189.79736328125,
      "logps/rejected": -241.90504455566406,
      "loss": 0.2008,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2981109619140625,
      "rewards/margins": 3.1607394218444824,
      "rewards/rejected": -4.458850860595703,
      "step": 1070
    },
    {
      "epoch": 1.4,
      "learning_rate": 4.798175609291161e-05,
      "logits/chosen": -2.4197938442230225,
      "logits/rejected": -2.3917999267578125,
      "logps/chosen": -212.80386352539062,
      "logps/rejected": -250.47430419921875,
      "loss": 0.2364,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.001808524131775,
      "rewards/margins": 3.6702208518981934,
      "rewards/rejected": -4.6720290184021,
      "step": 1071
    },
    {
      "epoch": 1.4,
      "learning_rate": 4.797765391347534e-05,
      "logits/chosen": -2.4398419857025146,
      "logits/rejected": -2.46700119972229,
      "logps/chosen": -183.11013793945312,
      "logps/rejected": -220.2843780517578,
      "loss": 0.1893,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9826763272285461,
      "rewards/margins": 3.509796142578125,
      "rewards/rejected": -4.4924726486206055,
      "step": 1072
    },
    {
      "epoch": 1.4,
      "learning_rate": 4.7973547745051074e-05,
      "logits/chosen": -2.4978957176208496,
      "logits/rejected": -2.5766334533691406,
      "logps/chosen": -219.82437133789062,
      "logps/rejected": -293.62359619140625,
      "loss": 0.146,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2996751070022583,
      "rewards/margins": 3.58370304107666,
      "rewards/rejected": -4.883378028869629,
      "step": 1073
    },
    {
      "epoch": 1.41,
      "learning_rate": 4.796943758835163e-05,
      "logits/chosen": -2.3775582313537598,
      "logits/rejected": -2.4787142276763916,
      "logps/chosen": -211.21597290039062,
      "logps/rejected": -254.6476287841797,
      "loss": 0.0379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7991434335708618,
      "rewards/margins": 4.452718734741211,
      "rewards/rejected": -5.251862049102783,
      "step": 1074
    },
    {
      "epoch": 1.41,
      "learning_rate": 4.796532344409055e-05,
      "logits/chosen": -2.4883899688720703,
      "logits/rejected": -2.5110130310058594,
      "logps/chosen": -183.7747802734375,
      "logps/rejected": -197.6341552734375,
      "loss": 0.1991,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0323280096054077,
      "rewards/margins": 2.9341001510620117,
      "rewards/rejected": -3.966428279876709,
      "step": 1075
    },
    {
      "epoch": 1.41,
      "learning_rate": 4.796120531298206e-05,
      "logits/chosen": -2.338197946548462,
      "logits/rejected": -2.2645106315612793,
      "logps/chosen": -201.51806640625,
      "logps/rejected": -266.6871032714844,
      "loss": 0.1477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9344640374183655,
      "rewards/margins": 3.64399790763855,
      "rewards/rejected": -4.57846212387085,
      "step": 1076
    },
    {
      "epoch": 1.41,
      "learning_rate": 4.795708319574109e-05,
      "logits/chosen": -2.5567445755004883,
      "logits/rejected": -2.5801315307617188,
      "logps/chosen": -264.55853271484375,
      "logps/rejected": -273.15447998046875,
      "loss": 0.1021,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.982978343963623,
      "rewards/margins": 4.4724040031433105,
      "rewards/rejected": -5.455382823944092,
      "step": 1077
    },
    {
      "epoch": 1.41,
      "learning_rate": 4.795295709308324e-05,
      "logits/chosen": -2.221895694732666,
      "logits/rejected": -2.1924304962158203,
      "logps/chosen": -199.84429931640625,
      "logps/rejected": -275.3081359863281,
      "loss": 0.1031,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0770325660705566,
      "rewards/margins": 4.073465347290039,
      "rewards/rejected": -5.150498390197754,
      "step": 1078
    },
    {
      "epoch": 1.41,
      "learning_rate": 4.7948827005724816e-05,
      "logits/chosen": -2.3237662315368652,
      "logits/rejected": -2.366284132003784,
      "logps/chosen": -159.32749938964844,
      "logps/rejected": -198.11053466796875,
      "loss": 0.2565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.221225380897522,
      "rewards/margins": 2.5992751121520996,
      "rewards/rejected": -3.820500373840332,
      "step": 1079
    },
    {
      "epoch": 1.41,
      "learning_rate": 4.794469293438282e-05,
      "logits/chosen": -2.443016767501831,
      "logits/rejected": -2.443124771118164,
      "logps/chosen": -185.0653076171875,
      "logps/rejected": -246.856689453125,
      "loss": 0.1283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7620292901992798,
      "rewards/margins": 3.877666711807251,
      "rewards/rejected": -4.639695644378662,
      "step": 1080
    },
    {
      "epoch": 1.41,
      "learning_rate": 4.7940554879774925e-05,
      "logits/chosen": -2.181255578994751,
      "logits/rejected": -2.305636167526245,
      "logps/chosen": -266.7353210449219,
      "logps/rejected": -283.93853759765625,
      "loss": 0.2479,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8880407810211182,
      "rewards/margins": 3.7701759338378906,
      "rewards/rejected": -4.65821647644043,
      "step": 1081
    },
    {
      "epoch": 1.42,
      "learning_rate": 4.793641284261953e-05,
      "logits/chosen": -2.369758367538452,
      "logits/rejected": -2.4988489151000977,
      "logps/chosen": -183.67445373535156,
      "logps/rejected": -233.878173828125,
      "loss": 0.1349,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2128649950027466,
      "rewards/margins": 3.4675493240356445,
      "rewards/rejected": -4.680413722991943,
      "step": 1082
    },
    {
      "epoch": 1.42,
      "learning_rate": 4.793226682363568e-05,
      "logits/chosen": -2.573646068572998,
      "logits/rejected": -2.576871395111084,
      "logps/chosen": -166.67950439453125,
      "logps/rejected": -206.69732666015625,
      "loss": 0.2356,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9300564527511597,
      "rewards/margins": 3.6901583671569824,
      "rewards/rejected": -4.620214939117432,
      "step": 1083
    },
    {
      "epoch": 1.42,
      "learning_rate": 4.7928116823543155e-05,
      "logits/chosen": -2.3677217960357666,
      "logits/rejected": -2.405235528945923,
      "logps/chosen": -187.89686584472656,
      "logps/rejected": -253.44918823242188,
      "loss": 0.1928,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2416791915893555,
      "rewards/margins": 3.303443670272827,
      "rewards/rejected": -4.545123100280762,
      "step": 1084
    },
    {
      "epoch": 1.42,
      "learning_rate": 4.79239628430624e-05,
      "logits/chosen": -2.4083261489868164,
      "logits/rejected": -2.4456753730773926,
      "logps/chosen": -216.77603149414062,
      "logps/rejected": -276.124755859375,
      "loss": 0.0495,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0237876176834106,
      "rewards/margins": 4.667983531951904,
      "rewards/rejected": -5.691771030426025,
      "step": 1085
    },
    {
      "epoch": 1.42,
      "learning_rate": 4.791980488291456e-05,
      "logits/chosen": -2.082496404647827,
      "logits/rejected": -2.0862114429473877,
      "logps/chosen": -197.0124969482422,
      "logps/rejected": -200.89968872070312,
      "loss": 0.1189,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8787529468536377,
      "rewards/margins": 3.3207461833953857,
      "rewards/rejected": -4.199499130249023,
      "step": 1086
    },
    {
      "epoch": 1.42,
      "learning_rate": 4.791564294382147e-05,
      "logits/chosen": -2.251490831375122,
      "logits/rejected": -2.3277134895324707,
      "logps/chosen": -176.1337432861328,
      "logps/rejected": -240.8201904296875,
      "loss": 0.112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4940316677093506,
      "rewards/margins": 4.046961784362793,
      "rewards/rejected": -4.5409932136535645,
      "step": 1087
    },
    {
      "epoch": 1.42,
      "learning_rate": 4.7911477026505654e-05,
      "logits/chosen": -2.273149251937866,
      "logits/rejected": -2.302190065383911,
      "logps/chosen": -198.4782257080078,
      "logps/rejected": -251.56442260742188,
      "loss": 0.1584,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0771201848983765,
      "rewards/margins": 4.303647994995117,
      "rewards/rejected": -5.380767822265625,
      "step": 1088
    },
    {
      "epoch": 1.43,
      "learning_rate": 4.790730713169033e-05,
      "logits/chosen": -2.35538649559021,
      "logits/rejected": -2.484128475189209,
      "logps/chosen": -200.436767578125,
      "logps/rejected": -264.001708984375,
      "loss": 0.088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3317238092422485,
      "rewards/margins": 3.762491226196289,
      "rewards/rejected": -5.094214916229248,
      "step": 1089
    },
    {
      "epoch": 1.43,
      "learning_rate": 4.7903133260099385e-05,
      "logits/chosen": -2.3187928199768066,
      "logits/rejected": -2.248053550720215,
      "logps/chosen": -269.09881591796875,
      "logps/rejected": -317.9813232421875,
      "loss": 0.1617,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5169427394866943,
      "rewards/margins": 4.865277290344238,
      "rewards/rejected": -6.382220268249512,
      "step": 1090
    },
    {
      "epoch": 1.43,
      "learning_rate": 4.789895541245745e-05,
      "logits/chosen": -2.2053849697113037,
      "logits/rejected": -2.249541759490967,
      "logps/chosen": -211.03778076171875,
      "logps/rejected": -254.56468200683594,
      "loss": 0.1818,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5010100603103638,
      "rewards/margins": 3.9072840213775635,
      "rewards/rejected": -5.408294200897217,
      "step": 1091
    },
    {
      "epoch": 1.43,
      "learning_rate": 4.7894773589489775e-05,
      "logits/chosen": -2.25164794921875,
      "logits/rejected": -2.364285945892334,
      "logps/chosen": -188.2033233642578,
      "logps/rejected": -213.60580444335938,
      "loss": 0.1675,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2163310050964355,
      "rewards/margins": 3.9235103130340576,
      "rewards/rejected": -5.139841079711914,
      "step": 1092
    },
    {
      "epoch": 1.43,
      "learning_rate": 4.7890587791922364e-05,
      "logits/chosen": -2.3585565090179443,
      "logits/rejected": -2.3559505939483643,
      "logps/chosen": -183.8409423828125,
      "logps/rejected": -233.60438537597656,
      "loss": 0.2356,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3878989219665527,
      "rewards/margins": 2.798985719680786,
      "rewards/rejected": -4.186884880065918,
      "step": 1093
    },
    {
      "epoch": 1.43,
      "learning_rate": 4.788639802048187e-05,
      "logits/chosen": -2.4514660835266113,
      "logits/rejected": -2.468566417694092,
      "logps/chosen": -189.787841796875,
      "logps/rejected": -232.9893798828125,
      "loss": 0.1471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1803812980651855,
      "rewards/margins": 3.827017307281494,
      "rewards/rejected": -5.00739860534668,
      "step": 1094
    },
    {
      "epoch": 1.43,
      "learning_rate": 4.788220427589566e-05,
      "logits/chosen": -2.278841257095337,
      "logits/rejected": -2.3293325901031494,
      "logps/chosen": -177.0409393310547,
      "logps/rejected": -221.64451599121094,
      "loss": 0.2064,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7956516742706299,
      "rewards/margins": 4.03370475769043,
      "rewards/rejected": -4.8293561935424805,
      "step": 1095
    },
    {
      "epoch": 1.43,
      "learning_rate": 4.787800655889176e-05,
      "logits/chosen": -2.2903358936309814,
      "logits/rejected": -2.3189077377319336,
      "logps/chosen": -185.59381103515625,
      "logps/rejected": -257.0823059082031,
      "loss": 0.178,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8907773494720459,
      "rewards/margins": 4.005603790283203,
      "rewards/rejected": -4.896381378173828,
      "step": 1096
    },
    {
      "epoch": 1.44,
      "learning_rate": 4.787380487019893e-05,
      "logits/chosen": -2.3953933715820312,
      "logits/rejected": -2.450338363647461,
      "logps/chosen": -179.10519409179688,
      "logps/rejected": -219.1466522216797,
      "loss": 0.1598,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0823649168014526,
      "rewards/margins": 3.8283493518829346,
      "rewards/rejected": -4.910714149475098,
      "step": 1097
    },
    {
      "epoch": 1.44,
      "learning_rate": 4.786959921054659e-05,
      "logits/chosen": -2.3358683586120605,
      "logits/rejected": -2.2801101207733154,
      "logps/chosen": -209.03997802734375,
      "logps/rejected": -293.61419677734375,
      "loss": 0.1063,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2742486000061035,
      "rewards/margins": 3.9310548305511475,
      "rewards/rejected": -5.205303192138672,
      "step": 1098
    },
    {
      "epoch": 1.44,
      "learning_rate": 4.7865389580664844e-05,
      "logits/chosen": -2.279576301574707,
      "logits/rejected": -2.350735664367676,
      "logps/chosen": -146.66612243652344,
      "logps/rejected": -216.35707092285156,
      "loss": 0.1297,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8987219929695129,
      "rewards/margins": 3.8822243213653564,
      "rewards/rejected": -4.780946254730225,
      "step": 1099
    },
    {
      "epoch": 1.44,
      "learning_rate": 4.78611759812845e-05,
      "logits/chosen": -2.109327554702759,
      "logits/rejected": -2.0962979793548584,
      "logps/chosen": -180.84161376953125,
      "logps/rejected": -209.8188018798828,
      "loss": 0.3155,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6152057647705078,
      "rewards/margins": 3.317700147628784,
      "rewards/rejected": -3.932906150817871,
      "step": 1100
    },
    {
      "epoch": 1.44,
      "learning_rate": 4.785695841313706e-05,
      "logits/chosen": -2.3722269535064697,
      "logits/rejected": -2.3200840950012207,
      "logps/chosen": -201.63294982910156,
      "logps/rejected": -254.41258239746094,
      "loss": 0.1841,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.35667884349823,
      "rewards/margins": 4.864499092102051,
      "rewards/rejected": -6.2211785316467285,
      "step": 1101
    },
    {
      "epoch": 1.44,
      "learning_rate": 4.785273687695469e-05,
      "logits/chosen": -2.3536105155944824,
      "logits/rejected": -2.4441490173339844,
      "logps/chosen": -174.35108947753906,
      "logps/rejected": -237.75894165039062,
      "loss": 0.116,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.450251579284668,
      "rewards/margins": 4.218486785888672,
      "rewards/rejected": -5.66873836517334,
      "step": 1102
    },
    {
      "epoch": 1.44,
      "learning_rate": 4.784851137347028e-05,
      "logits/chosen": -2.1573073863983154,
      "logits/rejected": -2.2236359119415283,
      "logps/chosen": -198.5815887451172,
      "logps/rejected": -232.4602813720703,
      "loss": 0.1035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0396865606307983,
      "rewards/margins": 3.5235390663146973,
      "rewards/rejected": -4.563225269317627,
      "step": 1103
    },
    {
      "epoch": 1.44,
      "learning_rate": 4.7844281903417376e-05,
      "logits/chosen": -2.0304176807403564,
      "logits/rejected": -2.100639820098877,
      "logps/chosen": -198.72247314453125,
      "logps/rejected": -221.755615234375,
      "loss": 0.2035,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3415790796279907,
      "rewards/margins": 2.7209715843200684,
      "rewards/rejected": -4.0625505447387695,
      "step": 1104
    },
    {
      "epoch": 1.45,
      "learning_rate": 4.784004846753023e-05,
      "logits/chosen": -2.074061393737793,
      "logits/rejected": -2.121853828430176,
      "logps/chosen": -166.87387084960938,
      "logps/rejected": -207.54183959960938,
      "loss": 0.2033,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6906567215919495,
      "rewards/margins": 4.180294513702393,
      "rewards/rejected": -4.870951175689697,
      "step": 1105
    },
    {
      "epoch": 1.45,
      "learning_rate": 4.783581106654378e-05,
      "logits/chosen": -2.173704147338867,
      "logits/rejected": -2.194322109222412,
      "logps/chosen": -160.7693328857422,
      "logps/rejected": -202.16580200195312,
      "loss": 0.2204,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3304364681243896,
      "rewards/margins": 2.7709617614746094,
      "rewards/rejected": -4.101398468017578,
      "step": 1106
    },
    {
      "epoch": 1.45,
      "learning_rate": 4.783156970119366e-05,
      "logits/chosen": -1.8855363130569458,
      "logits/rejected": -1.914825677871704,
      "logps/chosen": -188.28408813476562,
      "logps/rejected": -240.01547241210938,
      "loss": 0.2478,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0375070571899414,
      "rewards/margins": 3.5554230213165283,
      "rewards/rejected": -4.592930793762207,
      "step": 1107
    },
    {
      "epoch": 1.45,
      "learning_rate": 4.782732437221616e-05,
      "logits/chosen": -2.5382192134857178,
      "logits/rejected": -2.4349591732025146,
      "logps/chosen": -257.2680969238281,
      "logps/rejected": -267.1951904296875,
      "loss": 0.1059,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0496553182601929,
      "rewards/margins": 4.491069316864014,
      "rewards/rejected": -5.540725231170654,
      "step": 1108
    },
    {
      "epoch": 1.45,
      "learning_rate": 4.78230750803483e-05,
      "logits/chosen": -2.325007200241089,
      "logits/rejected": -2.4198029041290283,
      "logps/chosen": -162.60813903808594,
      "logps/rejected": -221.90960693359375,
      "loss": 0.1349,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.49094444513320923,
      "rewards/margins": 3.7173869609832764,
      "rewards/rejected": -4.208331108093262,
      "step": 1109
    },
    {
      "epoch": 1.45,
      "learning_rate": 4.781882182632776e-05,
      "logits/chosen": -2.475273370742798,
      "logits/rejected": -2.397214412689209,
      "logps/chosen": -206.89404296875,
      "logps/rejected": -258.7071838378906,
      "loss": 0.1856,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.805166482925415,
      "rewards/margins": 4.610111713409424,
      "rewards/rejected": -5.415278434753418,
      "step": 1110
    },
    {
      "epoch": 1.45,
      "learning_rate": 4.781456461089294e-05,
      "logits/chosen": -2.5161237716674805,
      "logits/rejected": -2.437736749649048,
      "logps/chosen": -248.8204345703125,
      "logps/rejected": -275.9759216308594,
      "loss": 0.1836,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.067987322807312,
      "rewards/margins": 4.009271144866943,
      "rewards/rejected": -5.0772576332092285,
      "step": 1111
    },
    {
      "epoch": 1.46,
      "learning_rate": 4.781030343478288e-05,
      "logits/chosen": -2.2793309688568115,
      "logits/rejected": -2.3616416454315186,
      "logps/chosen": -213.34039306640625,
      "logps/rejected": -262.52398681640625,
      "loss": 0.1304,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2378281354904175,
      "rewards/margins": 4.064054489135742,
      "rewards/rejected": -5.301882743835449,
      "step": 1112
    },
    {
      "epoch": 1.46,
      "learning_rate": 4.780603829873733e-05,
      "logits/chosen": -2.2147018909454346,
      "logits/rejected": -2.4757885932922363,
      "logps/chosen": -275.67681884765625,
      "logps/rejected": -306.8128662109375,
      "loss": 0.1641,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.911482810974121,
      "rewards/margins": 4.229495048522949,
      "rewards/rejected": -6.14097785949707,
      "step": 1113
    },
    {
      "epoch": 1.46,
      "learning_rate": 4.780176920349675e-05,
      "logits/chosen": -2.2692694664001465,
      "logits/rejected": -2.4222545623779297,
      "logps/chosen": -181.1542510986328,
      "logps/rejected": -243.48069763183594,
      "loss": 0.1874,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4185149669647217,
      "rewards/margins": 2.8215103149414062,
      "rewards/rejected": -4.240025520324707,
      "step": 1114
    },
    {
      "epoch": 1.46,
      "learning_rate": 4.7797496149802256e-05,
      "logits/chosen": -2.2589826583862305,
      "logits/rejected": -2.2955431938171387,
      "logps/chosen": -163.7628173828125,
      "logps/rejected": -257.77764892578125,
      "loss": 0.1802,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0072280168533325,
      "rewards/margins": 4.691400527954102,
      "rewards/rejected": -5.6986284255981445,
      "step": 1115
    },
    {
      "epoch": 1.46,
      "learning_rate": 4.779321913839566e-05,
      "logits/chosen": -2.281262159347534,
      "logits/rejected": -2.2819738388061523,
      "logps/chosen": -233.2173309326172,
      "logps/rejected": -272.6235656738281,
      "loss": 0.1662,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1393500566482544,
      "rewards/margins": 3.8536875247955322,
      "rewards/rejected": -4.993037700653076,
      "step": 1116
    },
    {
      "epoch": 1.46,
      "learning_rate": 4.778893817001948e-05,
      "logits/chosen": -2.2208681106567383,
      "logits/rejected": -2.1590828895568848,
      "logps/chosen": -179.7089080810547,
      "logps/rejected": -216.29774475097656,
      "loss": 0.1497,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.060105323791504,
      "rewards/margins": 3.382988214492798,
      "rewards/rejected": -4.443093776702881,
      "step": 1117
    },
    {
      "epoch": 1.46,
      "learning_rate": 4.7784653245416875e-05,
      "logits/chosen": -2.1777501106262207,
      "logits/rejected": -2.2831733226776123,
      "logps/chosen": -191.6259307861328,
      "logps/rejected": -239.21493530273438,
      "loss": 0.1255,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.195249080657959,
      "rewards/margins": 3.476778507232666,
      "rewards/rejected": -4.672027587890625,
      "step": 1118
    },
    {
      "epoch": 1.46,
      "learning_rate": 4.778036436533176e-05,
      "logits/chosen": -2.345694065093994,
      "logits/rejected": -2.400123357772827,
      "logps/chosen": -150.64157104492188,
      "logps/rejected": -217.12583923339844,
      "loss": 0.1818,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1695704460144043,
      "rewards/margins": 3.341207504272461,
      "rewards/rejected": -4.510777950286865,
      "step": 1119
    },
    {
      "epoch": 1.47,
      "learning_rate": 4.777607153050866e-05,
      "logits/chosen": -2.27506422996521,
      "logits/rejected": -2.3480772972106934,
      "logps/chosen": -226.47450256347656,
      "logps/rejected": -255.78909301757812,
      "loss": 0.1828,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.286346673965454,
      "rewards/margins": 3.472963333129883,
      "rewards/rejected": -4.759309768676758,
      "step": 1120
    },
    {
      "epoch": 1.47,
      "learning_rate": 4.7771774741692844e-05,
      "logits/chosen": -2.4806315898895264,
      "logits/rejected": -2.420682430267334,
      "logps/chosen": -182.59005737304688,
      "logps/rejected": -220.8089599609375,
      "loss": 0.1512,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9431337118148804,
      "rewards/margins": 3.128722906112671,
      "rewards/rejected": -4.071856498718262,
      "step": 1121
    },
    {
      "epoch": 1.47,
      "learning_rate": 4.776747399963024e-05,
      "logits/chosen": -2.4254984855651855,
      "logits/rejected": -2.4268956184387207,
      "logps/chosen": -181.3052215576172,
      "logps/rejected": -244.41795349121094,
      "loss": 0.1868,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8446401953697205,
      "rewards/margins": 3.5034520626068115,
      "rewards/rejected": -4.348092555999756,
      "step": 1122
    },
    {
      "epoch": 1.47,
      "learning_rate": 4.776316930506747e-05,
      "logits/chosen": -2.3528339862823486,
      "logits/rejected": -2.518993616104126,
      "logps/chosen": -204.74118041992188,
      "logps/rejected": -297.0326843261719,
      "loss": 0.0927,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9256747364997864,
      "rewards/margins": 4.606964111328125,
      "rewards/rejected": -5.5326385498046875,
      "step": 1123
    },
    {
      "epoch": 1.47,
      "learning_rate": 4.775886065875185e-05,
      "logits/chosen": -2.428316593170166,
      "logits/rejected": -2.4729697704315186,
      "logps/chosen": -192.39427185058594,
      "logps/rejected": -225.5529327392578,
      "loss": 0.1996,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1067038774490356,
      "rewards/margins": 2.738870620727539,
      "rewards/rejected": -3.845574378967285,
      "step": 1124
    },
    {
      "epoch": 1.47,
      "learning_rate": 4.775454806143137e-05,
      "logits/chosen": -2.3744990825653076,
      "logits/rejected": -2.2757740020751953,
      "logps/chosen": -174.80218505859375,
      "logps/rejected": -327.66351318359375,
      "loss": 0.0678,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8998466730117798,
      "rewards/margins": 4.191055774688721,
      "rewards/rejected": -5.090902805328369,
      "step": 1125
    },
    {
      "epoch": 1.47,
      "learning_rate": 4.775023151385469e-05,
      "logits/chosen": -2.5411629676818848,
      "logits/rejected": -2.5592217445373535,
      "logps/chosen": -221.6334228515625,
      "logps/rejected": -263.060546875,
      "loss": 0.1147,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.669825553894043,
      "rewards/margins": 3.485670804977417,
      "rewards/rejected": -5.155496597290039,
      "step": 1126
    },
    {
      "epoch": 1.47,
      "learning_rate": 4.774591101677121e-05,
      "logits/chosen": -2.322519540786743,
      "logits/rejected": -2.3820464611053467,
      "logps/chosen": -185.3223876953125,
      "logps/rejected": -212.63223266601562,
      "loss": 0.2022,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0957611799240112,
      "rewards/margins": 2.9892659187316895,
      "rewards/rejected": -4.085027694702148,
      "step": 1127
    },
    {
      "epoch": 1.48,
      "learning_rate": 4.774158657093095e-05,
      "logits/chosen": -2.4653425216674805,
      "logits/rejected": -2.5697944164276123,
      "logps/chosen": -164.1728973388672,
      "logps/rejected": -223.85023498535156,
      "loss": 0.1749,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4075347185134888,
      "rewards/margins": 3.1060991287231445,
      "rewards/rejected": -4.513633728027344,
      "step": 1128
    },
    {
      "epoch": 1.48,
      "learning_rate": 4.7737258177084665e-05,
      "logits/chosen": -2.2479147911071777,
      "logits/rejected": -2.271726369857788,
      "logps/chosen": -226.23324584960938,
      "logps/rejected": -316.6651306152344,
      "loss": 0.0813,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.880527138710022,
      "rewards/margins": 5.674856662750244,
      "rewards/rejected": -6.555383682250977,
      "step": 1129
    },
    {
      "epoch": 1.48,
      "learning_rate": 4.7732925835983775e-05,
      "logits/chosen": -2.1202504634857178,
      "logits/rejected": -2.141789436340332,
      "logps/chosen": -218.8267822265625,
      "logps/rejected": -285.7803039550781,
      "loss": 0.1359,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9516122341156006,
      "rewards/margins": 4.8950653076171875,
      "rewards/rejected": -5.846677780151367,
      "step": 1130
    },
    {
      "epoch": 1.48,
      "learning_rate": 4.772858954838038e-05,
      "logits/chosen": -2.4971561431884766,
      "logits/rejected": -2.5220043659210205,
      "logps/chosen": -226.0542755126953,
      "logps/rejected": -299.23779296875,
      "loss": 0.1074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.871485710144043,
      "rewards/margins": 5.10019588470459,
      "rewards/rejected": -5.971682071685791,
      "step": 1131
    },
    {
      "epoch": 1.48,
      "learning_rate": 4.772424931502727e-05,
      "logits/chosen": -2.4149010181427,
      "logits/rejected": -2.431765556335449,
      "logps/chosen": -280.7022399902344,
      "logps/rejected": -361.54388427734375,
      "loss": 0.1054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6145488023757935,
      "rewards/margins": 3.9257521629333496,
      "rewards/rejected": -5.5403008460998535,
      "step": 1132
    },
    {
      "epoch": 1.48,
      "learning_rate": 4.771990513667793e-05,
      "logits/chosen": -2.3434276580810547,
      "logits/rejected": -2.3973376750946045,
      "logps/chosen": -183.86436462402344,
      "logps/rejected": -243.69293212890625,
      "loss": 0.2449,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6164230108261108,
      "rewards/margins": 2.5519909858703613,
      "rewards/rejected": -4.168414115905762,
      "step": 1133
    },
    {
      "epoch": 1.48,
      "learning_rate": 4.771555701408652e-05,
      "logits/chosen": -2.5856552124023438,
      "logits/rejected": -2.4191064834594727,
      "logps/chosen": -237.94381713867188,
      "logps/rejected": -232.526611328125,
      "loss": 0.1426,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6252250671386719,
      "rewards/margins": 3.2301154136657715,
      "rewards/rejected": -3.8553402423858643,
      "step": 1134
    },
    {
      "epoch": 1.49,
      "learning_rate": 4.771120494800789e-05,
      "logits/chosen": -2.196472644805908,
      "logits/rejected": -2.1777446269989014,
      "logps/chosen": -198.78469848632812,
      "logps/rejected": -220.56103515625,
      "loss": 0.2213,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9573407173156738,
      "rewards/margins": 3.4439077377319336,
      "rewards/rejected": -4.401248455047607,
      "step": 1135
    },
    {
      "epoch": 1.49,
      "learning_rate": 4.7706848939197565e-05,
      "logits/chosen": -2.4557721614837646,
      "logits/rejected": -2.4789040088653564,
      "logps/chosen": -183.08238220214844,
      "logps/rejected": -270.4992370605469,
      "loss": 0.1262,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5791445970535278,
      "rewards/margins": 4.073019027709961,
      "rewards/rejected": -4.652163982391357,
      "step": 1136
    },
    {
      "epoch": 1.49,
      "learning_rate": 4.7702488988411765e-05,
      "logits/chosen": -2.4277684688568115,
      "logits/rejected": -2.4676928520202637,
      "logps/chosen": -206.0131378173828,
      "logps/rejected": -222.62411499023438,
      "loss": 0.2436,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7216240763664246,
      "rewards/margins": 2.8460915088653564,
      "rewards/rejected": -3.5677154064178467,
      "step": 1137
    },
    {
      "epoch": 1.49,
      "learning_rate": 4.7698125096407384e-05,
      "logits/chosen": -2.500568151473999,
      "logits/rejected": -2.5246546268463135,
      "logps/chosen": -237.37734985351562,
      "logps/rejected": -282.74267578125,
      "loss": 0.0789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6298863887786865,
      "rewards/margins": 4.078357219696045,
      "rewards/rejected": -4.7082438468933105,
      "step": 1138
    },
    {
      "epoch": 1.49,
      "learning_rate": 4.7693757263942015e-05,
      "logits/chosen": -2.5762946605682373,
      "logits/rejected": -2.543735980987549,
      "logps/chosen": -240.41571044921875,
      "logps/rejected": -255.56991577148438,
      "loss": 0.0945,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8246884346008301,
      "rewards/margins": 4.176553726196289,
      "rewards/rejected": -5.001242637634277,
      "step": 1139
    },
    {
      "epoch": 1.49,
      "learning_rate": 4.768938549177393e-05,
      "logits/chosen": -2.426650047302246,
      "logits/rejected": -2.5201663970947266,
      "logps/chosen": -195.04554748535156,
      "logps/rejected": -247.7749786376953,
      "loss": 0.0827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9046878814697266,
      "rewards/margins": 3.637204170227051,
      "rewards/rejected": -4.541892051696777,
      "step": 1140
    },
    {
      "epoch": 1.49,
      "learning_rate": 4.7685009780662074e-05,
      "logits/chosen": -2.114713191986084,
      "logits/rejected": -2.0542666912078857,
      "logps/chosen": -184.0598602294922,
      "logps/rejected": -212.22396850585938,
      "loss": 0.1166,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9158424735069275,
      "rewards/margins": 3.6370954513549805,
      "rewards/rejected": -4.5529375076293945,
      "step": 1141
    },
    {
      "epoch": 1.49,
      "learning_rate": 4.768063013136607e-05,
      "logits/chosen": -2.494703769683838,
      "logits/rejected": -2.525118112564087,
      "logps/chosen": -232.44784545898438,
      "logps/rejected": -251.73089599609375,
      "loss": 0.1935,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5020594596862793,
      "rewards/margins": 2.5554111003875732,
      "rewards/rejected": -4.057470321655273,
      "step": 1142
    },
    {
      "epoch": 1.5,
      "learning_rate": 4.7676246544646266e-05,
      "logits/chosen": -2.445399522781372,
      "logits/rejected": -2.4102258682250977,
      "logps/chosen": -188.2194061279297,
      "logps/rejected": -230.07611083984375,
      "loss": 0.1288,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5976011753082275,
      "rewards/margins": 3.648184299468994,
      "rewards/rejected": -4.245785713195801,
      "step": 1143
    },
    {
      "epoch": 1.5,
      "learning_rate": 4.767185902126364e-05,
      "logits/chosen": -2.579176664352417,
      "logits/rejected": -2.610398292541504,
      "logps/chosen": -194.1246795654297,
      "logps/rejected": -236.80584716796875,
      "loss": 0.1657,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4905986785888672,
      "rewards/margins": 3.536527395248413,
      "rewards/rejected": -5.027125835418701,
      "step": 1144
    },
    {
      "epoch": 1.5,
      "learning_rate": 4.766746756197989e-05,
      "logits/chosen": -2.4602108001708984,
      "logits/rejected": -2.4495668411254883,
      "logps/chosen": -234.55178833007812,
      "logps/rejected": -269.4857177734375,
      "loss": 0.3061,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2820649147033691,
      "rewards/margins": 2.805851936340332,
      "rewards/rejected": -4.087916851043701,
      "step": 1145
    },
    {
      "epoch": 1.5,
      "learning_rate": 4.766307216755739e-05,
      "logits/chosen": -2.402568817138672,
      "logits/rejected": -2.4603004455566406,
      "logps/chosen": -184.5412139892578,
      "logps/rejected": -232.99713134765625,
      "loss": 0.1395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6240134239196777,
      "rewards/margins": 3.225022077560425,
      "rewards/rejected": -4.849035263061523,
      "step": 1146
    },
    {
      "epoch": 1.5,
      "learning_rate": 4.765867283875919e-05,
      "logits/chosen": -2.3566904067993164,
      "logits/rejected": -2.315061092376709,
      "logps/chosen": -194.67453002929688,
      "logps/rejected": -208.98086547851562,
      "loss": 0.1716,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9168694019317627,
      "rewards/margins": 3.383730173110962,
      "rewards/rejected": -4.300599575042725,
      "step": 1147
    },
    {
      "epoch": 1.5,
      "learning_rate": 4.765426957634903e-05,
      "logits/chosen": -2.288248062133789,
      "logits/rejected": -2.3149735927581787,
      "logps/chosen": -248.4288330078125,
      "logps/rejected": -294.727294921875,
      "loss": 0.0246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7581281661987305,
      "rewards/margins": 4.977880954742432,
      "rewards/rejected": -5.73600959777832,
      "step": 1148
    },
    {
      "epoch": 1.5,
      "learning_rate": 4.7649862381091326e-05,
      "logits/chosen": -2.441120147705078,
      "logits/rejected": -2.507535934448242,
      "logps/chosen": -145.71258544921875,
      "logps/rejected": -208.50909423828125,
      "loss": 0.2425,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5849244594573975,
      "rewards/margins": 2.7225358486175537,
      "rewards/rejected": -4.307460308074951,
      "step": 1149
    },
    {
      "epoch": 1.5,
      "learning_rate": 4.764545125375117e-05,
      "logits/chosen": -2.0023226737976074,
      "logits/rejected": -2.0600125789642334,
      "logps/chosen": -152.4337615966797,
      "logps/rejected": -222.3336639404297,
      "loss": 0.1372,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2324447631835938,
      "rewards/margins": 3.39628267288208,
      "rewards/rejected": -4.628727436065674,
      "step": 1150
    },
    {
      "epoch": 1.51,
      "learning_rate": 4.764103619509436e-05,
      "logits/chosen": -2.4556169509887695,
      "logits/rejected": -2.412785530090332,
      "logps/chosen": -199.9222869873047,
      "logps/rejected": -212.75326538085938,
      "loss": 0.28,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5000001192092896,
      "rewards/margins": 3.5534515380859375,
      "rewards/rejected": -5.053452014923096,
      "step": 1151
    },
    {
      "epoch": 1.51,
      "learning_rate": 4.763661720588736e-05,
      "logits/chosen": -2.436502456665039,
      "logits/rejected": -2.4594924449920654,
      "logps/chosen": -209.86231994628906,
      "logps/rejected": -230.40859985351562,
      "loss": 0.1923,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5364642143249512,
      "rewards/margins": 3.644205093383789,
      "rewards/rejected": -5.18066930770874,
      "step": 1152
    },
    {
      "epoch": 1.51,
      "learning_rate": 4.7632194286897315e-05,
      "logits/chosen": -2.570007801055908,
      "logits/rejected": -2.5704219341278076,
      "logps/chosen": -221.38475036621094,
      "logps/rejected": -304.7647705078125,
      "loss": 0.1645,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2075974941253662,
      "rewards/margins": 5.129228591918945,
      "rewards/rejected": -6.336826324462891,
      "step": 1153
    },
    {
      "epoch": 1.51,
      "learning_rate": 4.762776743889207e-05,
      "logits/chosen": -2.6236629486083984,
      "logits/rejected": -2.678802967071533,
      "logps/chosen": -311.6545104980469,
      "logps/rejected": -333.4806823730469,
      "loss": 0.1045,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6849138736724854,
      "rewards/margins": 4.670904636383057,
      "rewards/rejected": -6.355818271636963,
      "step": 1154
    },
    {
      "epoch": 1.51,
      "learning_rate": 4.7623336662640116e-05,
      "logits/chosen": -1.7820888757705688,
      "logits/rejected": -1.8834837675094604,
      "logps/chosen": -162.43157958984375,
      "logps/rejected": -243.75802612304688,
      "loss": 0.0676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.156273365020752,
      "rewards/margins": 3.5598320960998535,
      "rewards/rejected": -5.716104984283447,
      "step": 1155
    },
    {
      "epoch": 1.51,
      "learning_rate": 4.761890195891067e-05,
      "logits/chosen": -2.580514907836914,
      "logits/rejected": -2.583678722381592,
      "logps/chosen": -227.94639587402344,
      "logps/rejected": -269.7225341796875,
      "loss": 0.0754,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5052528381347656,
      "rewards/margins": 4.785727024078369,
      "rewards/rejected": -6.290980815887451,
      "step": 1156
    },
    {
      "epoch": 1.51,
      "learning_rate": 4.76144633284736e-05,
      "logits/chosen": -2.2309510707855225,
      "logits/rejected": -2.2391128540039062,
      "logps/chosen": -204.0966033935547,
      "logps/rejected": -237.06265258789062,
      "loss": 0.2103,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.803736925125122,
      "rewards/margins": 2.9250359535217285,
      "rewards/rejected": -5.728772163391113,
      "step": 1157
    },
    {
      "epoch": 1.52,
      "learning_rate": 4.761002077209946e-05,
      "logits/chosen": -2.3017187118530273,
      "logits/rejected": -2.1956186294555664,
      "logps/chosen": -204.7859344482422,
      "logps/rejected": -202.06703186035156,
      "loss": 0.1633,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.668914794921875,
      "rewards/margins": 3.471302032470703,
      "rewards/rejected": -5.140216827392578,
      "step": 1158
    },
    {
      "epoch": 1.52,
      "learning_rate": 4.760557429055951e-05,
      "logits/chosen": -2.2529184818267822,
      "logits/rejected": -2.1692707538604736,
      "logps/chosen": -202.95797729492188,
      "logps/rejected": -222.89297485351562,
      "loss": 0.2165,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.9888824224472046,
      "rewards/margins": 3.733574390411377,
      "rewards/rejected": -5.722457408905029,
      "step": 1159
    },
    {
      "epoch": 1.52,
      "learning_rate": 4.760112388462564e-05,
      "logits/chosen": -2.5051965713500977,
      "logits/rejected": -2.461354970932007,
      "logps/chosen": -207.01870727539062,
      "logps/rejected": -233.17376708984375,
      "loss": 0.2913,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.4785971641540527,
      "rewards/margins": 3.71671199798584,
      "rewards/rejected": -6.195309638977051,
      "step": 1160
    },
    {
      "epoch": 1.52,
      "learning_rate": 4.759666955507049e-05,
      "logits/chosen": -2.3772454261779785,
      "logits/rejected": -2.3350820541381836,
      "logps/chosen": -194.571533203125,
      "logps/rejected": -220.87799072265625,
      "loss": 0.1533,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7106103897094727,
      "rewards/margins": 3.6144776344299316,
      "rewards/rejected": -5.325088024139404,
      "step": 1161
    },
    {
      "epoch": 1.52,
      "learning_rate": 4.759221130266732e-05,
      "logits/chosen": -2.2555718421936035,
      "logits/rejected": -2.2726223468780518,
      "logps/chosen": -175.2388153076172,
      "logps/rejected": -250.49134826660156,
      "loss": 0.091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.260037422180176,
      "rewards/margins": 4.190368175506592,
      "rewards/rejected": -6.450405120849609,
      "step": 1162
    },
    {
      "epoch": 1.52,
      "learning_rate": 4.758774912819011e-05,
      "logits/chosen": -2.5848538875579834,
      "logits/rejected": -2.6254186630249023,
      "logps/chosen": -295.5448913574219,
      "logps/rejected": -309.3749084472656,
      "loss": 0.0571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9205611944198608,
      "rewards/margins": 4.6831841468811035,
      "rewards/rejected": -6.603745460510254,
      "step": 1163
    },
    {
      "epoch": 1.52,
      "learning_rate": 4.758328303241349e-05,
      "logits/chosen": -2.445176124572754,
      "logits/rejected": -2.4309027194976807,
      "logps/chosen": -228.26736450195312,
      "logps/rejected": -228.83767700195312,
      "loss": 0.1921,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.568021297454834,
      "rewards/margins": 3.549888849258423,
      "rewards/rejected": -6.117909908294678,
      "step": 1164
    },
    {
      "epoch": 1.52,
      "learning_rate": 4.75788130161128e-05,
      "logits/chosen": -2.2904882431030273,
      "logits/rejected": -2.3325557708740234,
      "logps/chosen": -225.65245056152344,
      "logps/rejected": -264.3974304199219,
      "loss": 0.141,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.110083818435669,
      "rewards/margins": 4.307819366455078,
      "rewards/rejected": -6.417903423309326,
      "step": 1165
    },
    {
      "epoch": 1.53,
      "learning_rate": 4.7574339080064044e-05,
      "logits/chosen": -2.248126268386841,
      "logits/rejected": -2.4054722785949707,
      "logps/chosen": -179.7675323486328,
      "logps/rejected": -227.87307739257812,
      "loss": 0.1646,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8558435440063477,
      "rewards/margins": 3.661419153213501,
      "rewards/rejected": -5.5172624588012695,
      "step": 1166
    },
    {
      "epoch": 1.53,
      "learning_rate": 4.756986122504392e-05,
      "logits/chosen": -2.5831997394561768,
      "logits/rejected": -2.652710199356079,
      "logps/chosen": -187.14601135253906,
      "logps/rejected": -252.51132202148438,
      "loss": 0.0897,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8020074367523193,
      "rewards/margins": 4.59144401550293,
      "rewards/rejected": -6.393451690673828,
      "step": 1167
    },
    {
      "epoch": 1.53,
      "learning_rate": 4.756537945182978e-05,
      "logits/chosen": -2.3583719730377197,
      "logits/rejected": -2.4774858951568604,
      "logps/chosen": -182.31289672851562,
      "logps/rejected": -262.7928161621094,
      "loss": 0.0801,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.851448655128479,
      "rewards/margins": 4.432652950286865,
      "rewards/rejected": -6.284101486206055,
      "step": 1168
    },
    {
      "epoch": 1.53,
      "learning_rate": 4.7560893761199685e-05,
      "logits/chosen": -2.4317312240600586,
      "logits/rejected": -2.548471689224243,
      "logps/chosen": -224.8054656982422,
      "logps/rejected": -264.10919189453125,
      "loss": 0.2068,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.833303213119507,
      "rewards/margins": 3.181811809539795,
      "rewards/rejected": -6.015115261077881,
      "step": 1169
    },
    {
      "epoch": 1.53,
      "learning_rate": 4.7556404153932356e-05,
      "logits/chosen": -2.501443386077881,
      "logits/rejected": -2.562654495239258,
      "logps/chosen": -186.91287231445312,
      "logps/rejected": -231.56109619140625,
      "loss": 0.0915,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9190934896469116,
      "rewards/margins": 4.339428424835205,
      "rewards/rejected": -6.258522033691406,
      "step": 1170
    },
    {
      "epoch": 1.53,
      "learning_rate": 4.755191063080721e-05,
      "logits/chosen": -2.152017116546631,
      "logits/rejected": -2.183455467224121,
      "logps/chosen": -195.06324768066406,
      "logps/rejected": -249.9405517578125,
      "loss": 0.1957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.957331657409668,
      "rewards/margins": 3.673862934112549,
      "rewards/rejected": -5.631194591522217,
      "step": 1171
    },
    {
      "epoch": 1.53,
      "learning_rate": 4.754741319260433e-05,
      "logits/chosen": -2.1391313076019287,
      "logits/rejected": -2.259127616882324,
      "logps/chosen": -218.9195098876953,
      "logps/rejected": -249.30020141601562,
      "loss": 0.1433,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.8016879558563232,
      "rewards/margins": 4.185688018798828,
      "rewards/rejected": -6.9873762130737305,
      "step": 1172
    },
    {
      "epoch": 1.54,
      "learning_rate": 4.754291184010449e-05,
      "logits/chosen": -2.3584299087524414,
      "logits/rejected": -2.4219892024993896,
      "logps/chosen": -250.45458984375,
      "logps/rejected": -282.7275390625,
      "loss": 0.1347,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.450329303741455,
      "rewards/margins": 3.8380470275878906,
      "rewards/rejected": -6.288376808166504,
      "step": 1173
    },
    {
      "epoch": 1.54,
      "learning_rate": 4.753840657408913e-05,
      "logits/chosen": -2.546412706375122,
      "logits/rejected": -2.5415077209472656,
      "logps/chosen": -218.136474609375,
      "logps/rejected": -255.75633239746094,
      "loss": 0.0751,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4324724674224854,
      "rewards/margins": 4.358306884765625,
      "rewards/rejected": -5.790779113769531,
      "step": 1174
    },
    {
      "epoch": 1.54,
      "learning_rate": 4.7533897395340384e-05,
      "logits/chosen": -2.3359920978546143,
      "logits/rejected": -2.2689504623413086,
      "logps/chosen": -175.5142364501953,
      "logps/rejected": -207.70046997070312,
      "loss": 0.2638,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7872010469436646,
      "rewards/margins": 3.380298376083374,
      "rewards/rejected": -5.167499542236328,
      "step": 1175
    },
    {
      "epoch": 1.54,
      "learning_rate": 4.752938430464105e-05,
      "logits/chosen": -2.5895543098449707,
      "logits/rejected": -2.645479679107666,
      "logps/chosen": -250.36102294921875,
      "logps/rejected": -294.0989074707031,
      "loss": 0.0946,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3163928985595703,
      "rewards/margins": 5.06205940246582,
      "rewards/rejected": -6.378452301025391,
      "step": 1176
    },
    {
      "epoch": 1.54,
      "learning_rate": 4.752486730277463e-05,
      "logits/chosen": -1.9727551937103271,
      "logits/rejected": -2.0490756034851074,
      "logps/chosen": -183.3731231689453,
      "logps/rejected": -266.77679443359375,
      "loss": 0.2344,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9805545806884766,
      "rewards/margins": 3.784672975540161,
      "rewards/rejected": -5.765227317810059,
      "step": 1177
    },
    {
      "epoch": 1.54,
      "learning_rate": 4.752034639052527e-05,
      "logits/chosen": -2.3057782649993896,
      "logits/rejected": -2.363804817199707,
      "logps/chosen": -210.0028533935547,
      "logps/rejected": -274.5212707519531,
      "loss": 0.1186,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.943210244178772,
      "rewards/margins": 4.820004463195801,
      "rewards/rejected": -6.763214588165283,
      "step": 1178
    },
    {
      "epoch": 1.54,
      "learning_rate": 4.751582156867783e-05,
      "logits/chosen": -2.40464186668396,
      "logits/rejected": -2.3199591636657715,
      "logps/chosen": -238.06265258789062,
      "logps/rejected": -262.56134033203125,
      "loss": 0.2005,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6629893779754639,
      "rewards/margins": 3.4837026596069336,
      "rewards/rejected": -5.146691799163818,
      "step": 1179
    },
    {
      "epoch": 1.54,
      "learning_rate": 4.751129283801782e-05,
      "logits/chosen": -2.4328413009643555,
      "logits/rejected": -2.546642780303955,
      "logps/chosen": -248.2761993408203,
      "logps/rejected": -308.90130615234375,
      "loss": 0.0414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.027224540710449,
      "rewards/margins": 5.0925612449646,
      "rewards/rejected": -7.119786262512207,
      "step": 1180
    },
    {
      "epoch": 1.55,
      "learning_rate": 4.7506760199331445e-05,
      "logits/chosen": -2.329103469848633,
      "logits/rejected": -2.442920207977295,
      "logps/chosen": -196.090087890625,
      "logps/rejected": -244.14735412597656,
      "loss": 0.133,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.5378122329711914,
      "rewards/margins": 3.5418591499328613,
      "rewards/rejected": -6.079671859741211,
      "step": 1181
    },
    {
      "epoch": 1.55,
      "learning_rate": 4.750222365340559e-05,
      "logits/chosen": -2.4741384983062744,
      "logits/rejected": -2.5175294876098633,
      "logps/chosen": -194.5352325439453,
      "logps/rejected": -271.7839660644531,
      "loss": 0.1235,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9755550622940063,
      "rewards/margins": 4.500469207763672,
      "rewards/rejected": -6.476024627685547,
      "step": 1182
    },
    {
      "epoch": 1.55,
      "learning_rate": 4.749768320102781e-05,
      "logits/chosen": -2.535355806350708,
      "logits/rejected": -2.596473217010498,
      "logps/chosen": -246.87472534179688,
      "logps/rejected": -301.79296875,
      "loss": 0.0759,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.202014446258545,
      "rewards/margins": 4.9576096534729,
      "rewards/rejected": -6.159624099731445,
      "step": 1183
    },
    {
      "epoch": 1.55,
      "learning_rate": 4.749313884298633e-05,
      "logits/chosen": -2.47165846824646,
      "logits/rejected": -2.4970316886901855,
      "logps/chosen": -244.66204833984375,
      "logps/rejected": -291.098388671875,
      "loss": 0.213,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.790406584739685,
      "rewards/margins": 5.0259270668029785,
      "rewards/rejected": -6.816333770751953,
      "step": 1184
    },
    {
      "epoch": 1.55,
      "learning_rate": 4.7488590580070074e-05,
      "logits/chosen": -2.3476386070251465,
      "logits/rejected": -2.4502997398376465,
      "logps/chosen": -223.2727508544922,
      "logps/rejected": -293.4349365234375,
      "loss": 0.1489,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.024826765060425,
      "rewards/margins": 3.9169485569000244,
      "rewards/rejected": -5.941775321960449,
      "step": 1185
    },
    {
      "epoch": 1.55,
      "learning_rate": 4.748403841306863e-05,
      "logits/chosen": -2.0279314517974854,
      "logits/rejected": -2.0650389194488525,
      "logps/chosen": -163.14549255371094,
      "logps/rejected": -210.16258239746094,
      "loss": 0.4483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0485494136810303,
      "rewards/margins": 2.879338264465332,
      "rewards/rejected": -4.927887916564941,
      "step": 1186
    },
    {
      "epoch": 1.55,
      "learning_rate": 4.747948234277228e-05,
      "logits/chosen": -2.5351710319519043,
      "logits/rejected": -2.5304994583129883,
      "logps/chosen": -267.7850036621094,
      "logps/rejected": -258.6395263671875,
      "loss": 0.1651,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5308005809783936,
      "rewards/margins": 4.033406734466553,
      "rewards/rejected": -5.564207077026367,
      "step": 1187
    },
    {
      "epoch": 1.55,
      "learning_rate": 4.747492236997195e-05,
      "logits/chosen": -2.3023691177368164,
      "logits/rejected": -2.3658742904663086,
      "logps/chosen": -192.9929656982422,
      "logps/rejected": -237.79534912109375,
      "loss": 0.2237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.313674807548523,
      "rewards/margins": 3.4801218509674072,
      "rewards/rejected": -4.793796539306641,
      "step": 1188
    },
    {
      "epoch": 1.56,
      "learning_rate": 4.747035849545928e-05,
      "logits/chosen": -2.5478949546813965,
      "logits/rejected": -2.51932954788208,
      "logps/chosen": -239.89755249023438,
      "logps/rejected": -269.83544921875,
      "loss": 0.0833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7208430767059326,
      "rewards/margins": 4.114684104919434,
      "rewards/rejected": -5.835526943206787,
      "step": 1189
    },
    {
      "epoch": 1.56,
      "learning_rate": 4.746579072002657e-05,
      "logits/chosen": -2.2504239082336426,
      "logits/rejected": -2.3190526962280273,
      "logps/chosen": -224.220703125,
      "logps/rejected": -260.7024841308594,
      "loss": 0.3915,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0131428241729736,
      "rewards/margins": 2.687302589416504,
      "rewards/rejected": -4.700445652008057,
      "step": 1190
    },
    {
      "epoch": 1.56,
      "learning_rate": 4.7461219044466795e-05,
      "logits/chosen": -2.1805527210235596,
      "logits/rejected": -2.193903684616089,
      "logps/chosen": -184.85508728027344,
      "logps/rejected": -234.97364807128906,
      "loss": 0.2377,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2396941184997559,
      "rewards/margins": 3.461569309234619,
      "rewards/rejected": -4.701263427734375,
      "step": 1191
    },
    {
      "epoch": 1.56,
      "learning_rate": 4.745664346957361e-05,
      "logits/chosen": -2.694643974304199,
      "logits/rejected": -2.705972194671631,
      "logps/chosen": -231.14852905273438,
      "logps/rejected": -266.03460693359375,
      "loss": 0.189,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.598564624786377,
      "rewards/margins": 3.681683301925659,
      "rewards/rejected": -6.280247688293457,
      "step": 1192
    },
    {
      "epoch": 1.56,
      "learning_rate": 4.745206399614135e-05,
      "logits/chosen": -2.131896734237671,
      "logits/rejected": -2.225205421447754,
      "logps/chosen": -222.63446044921875,
      "logps/rejected": -309.03814697265625,
      "loss": 0.1005,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2715827226638794,
      "rewards/margins": 4.563998222351074,
      "rewards/rejected": -5.835580825805664,
      "step": 1193
    },
    {
      "epoch": 1.56,
      "learning_rate": 4.744748062496503e-05,
      "logits/chosen": -2.4835543632507324,
      "logits/rejected": -2.552267551422119,
      "logps/chosen": -166.58905029296875,
      "logps/rejected": -213.53118896484375,
      "loss": 0.2019,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.377805233001709,
      "rewards/margins": 3.6189918518066406,
      "rewards/rejected": -4.99679708480835,
      "step": 1194
    },
    {
      "epoch": 1.56,
      "learning_rate": 4.744289335684034e-05,
      "logits/chosen": -2.522873640060425,
      "logits/rejected": -2.517055034637451,
      "logps/chosen": -203.4969482421875,
      "logps/rejected": -267.63873291015625,
      "loss": 0.1573,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.246862530708313,
      "rewards/margins": 4.703207492828369,
      "rewards/rejected": -5.950070381164551,
      "step": 1195
    },
    {
      "epoch": 1.57,
      "learning_rate": 4.7438302192563625e-05,
      "logits/chosen": -2.3815245628356934,
      "logits/rejected": -2.4553897380828857,
      "logps/chosen": -231.88101196289062,
      "logps/rejected": -279.2082824707031,
      "loss": 0.1052,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3264628648757935,
      "rewards/margins": 4.349924564361572,
      "rewards/rejected": -5.676387786865234,
      "step": 1196
    },
    {
      "epoch": 1.57,
      "learning_rate": 4.743370713293194e-05,
      "logits/chosen": -2.671739101409912,
      "logits/rejected": -2.6977474689483643,
      "logps/chosen": -245.71820068359375,
      "logps/rejected": -263.50811767578125,
      "loss": 0.2618,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.2178151607513428,
      "rewards/margins": 3.918609142303467,
      "rewards/rejected": -6.1364240646362305,
      "step": 1197
    },
    {
      "epoch": 1.57,
      "learning_rate": 4.7429108178742985e-05,
      "logits/chosen": -2.3009557723999023,
      "logits/rejected": -2.2271933555603027,
      "logps/chosen": -181.82748413085938,
      "logps/rejected": -212.26087951660156,
      "loss": 0.5133,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7228325605392456,
      "rewards/margins": 3.406215190887451,
      "rewards/rejected": -5.129047870635986,
      "step": 1198
    },
    {
      "epoch": 1.57,
      "learning_rate": 4.742450533079518e-05,
      "logits/chosen": -2.5072877407073975,
      "logits/rejected": -2.4657020568847656,
      "logps/chosen": -183.00059509277344,
      "logps/rejected": -212.53353881835938,
      "loss": 0.3148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.878244161605835,
      "rewards/margins": 2.4889166355133057,
      "rewards/rejected": -4.367160797119141,
      "step": 1199
    },
    {
      "epoch": 1.57,
      "learning_rate": 4.7419898589887566e-05,
      "logits/chosen": -2.134070873260498,
      "logits/rejected": -2.256047487258911,
      "logps/chosen": -170.70077514648438,
      "logps/rejected": -231.8829345703125,
      "loss": 0.1302,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0568268299102783,
      "rewards/margins": 4.093639373779297,
      "rewards/rejected": -5.1504669189453125,
      "step": 1200
    },
    {
      "epoch": 1.57,
      "learning_rate": 4.74152879568199e-05,
      "logits/chosen": -2.6162269115448,
      "logits/rejected": -2.652150869369507,
      "logps/chosen": -233.90402221679688,
      "logps/rejected": -244.74334716796875,
      "loss": 0.0997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6784412860870361,
      "rewards/margins": 3.7588188648223877,
      "rewards/rejected": -5.437260627746582,
      "step": 1201
    },
    {
      "epoch": 1.57,
      "learning_rate": 4.7410673432392596e-05,
      "logits/chosen": -2.484144687652588,
      "logits/rejected": -2.5014195442199707,
      "logps/chosen": -250.83456420898438,
      "logps/rejected": -281.1622314453125,
      "loss": 0.2326,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0612598657608032,
      "rewards/margins": 3.5049850940704346,
      "rewards/rejected": -4.566245079040527,
      "step": 1202
    },
    {
      "epoch": 1.57,
      "learning_rate": 4.7406055017406754e-05,
      "logits/chosen": -2.6265833377838135,
      "logits/rejected": -2.7116756439208984,
      "logps/chosen": -236.57855224609375,
      "logps/rejected": -276.8167724609375,
      "loss": 0.1522,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.490135669708252,
      "rewards/margins": 3.229520082473755,
      "rewards/rejected": -4.719655513763428,
      "step": 1203
    },
    {
      "epoch": 1.58,
      "learning_rate": 4.740143271266414e-05,
      "logits/chosen": -2.5222041606903076,
      "logits/rejected": -2.6354575157165527,
      "logps/chosen": -229.79931640625,
      "logps/rejected": -270.754638671875,
      "loss": 0.0646,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0310989618301392,
      "rewards/margins": 5.396121025085449,
      "rewards/rejected": -6.427220344543457,
      "step": 1204
    },
    {
      "epoch": 1.58,
      "learning_rate": 4.73968065189672e-05,
      "logits/chosen": -2.396852731704712,
      "logits/rejected": -2.4251794815063477,
      "logps/chosen": -192.14022827148438,
      "logps/rejected": -229.5162353515625,
      "loss": 0.3256,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.227713704109192,
      "rewards/margins": 2.811344861984253,
      "rewards/rejected": -4.039058685302734,
      "step": 1205
    },
    {
      "epoch": 1.58,
      "learning_rate": 4.739217643711906e-05,
      "logits/chosen": -2.287642478942871,
      "logits/rejected": -2.3326570987701416,
      "logps/chosen": -164.69529724121094,
      "logps/rejected": -231.8920135498047,
      "loss": 0.3453,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.245210886001587,
      "rewards/margins": 2.8619096279144287,
      "rewards/rejected": -4.107120513916016,
      "step": 1206
    },
    {
      "epoch": 1.58,
      "learning_rate": 4.7387542467923505e-05,
      "logits/chosen": -2.479602575302124,
      "logits/rejected": -2.5209507942199707,
      "logps/chosen": -244.21514892578125,
      "logps/rejected": -303.6318664550781,
      "loss": 0.0309,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2694910764694214,
      "rewards/margins": 5.570847034454346,
      "rewards/rejected": -6.840338706970215,
      "step": 1207
    },
    {
      "epoch": 1.58,
      "learning_rate": 4.738290461218502e-05,
      "logits/chosen": -2.4771368503570557,
      "logits/rejected": -2.456359386444092,
      "logps/chosen": -230.273681640625,
      "logps/rejected": -269.0933837890625,
      "loss": 0.1615,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9786639213562012,
      "rewards/margins": 3.434558629989624,
      "rewards/rejected": -5.413222312927246,
      "step": 1208
    },
    {
      "epoch": 1.58,
      "learning_rate": 4.737826287070874e-05,
      "logits/chosen": -2.2615694999694824,
      "logits/rejected": -2.326125383377075,
      "logps/chosen": -157.7958526611328,
      "logps/rejected": -257.5984191894531,
      "loss": 0.1683,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3305480480194092,
      "rewards/margins": 3.8238983154296875,
      "rewards/rejected": -5.154446125030518,
      "step": 1209
    },
    {
      "epoch": 1.58,
      "learning_rate": 4.737361724430048e-05,
      "logits/chosen": -2.5612540245056152,
      "logits/rejected": -2.6601011753082275,
      "logps/chosen": -240.48211669921875,
      "logps/rejected": -257.5409851074219,
      "loss": 0.2828,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7493575811386108,
      "rewards/margins": 2.682534694671631,
      "rewards/rejected": -4.431892395019531,
      "step": 1210
    },
    {
      "epoch": 1.58,
      "learning_rate": 4.7368967733766756e-05,
      "logits/chosen": -2.0791330337524414,
      "logits/rejected": -2.0077402591705322,
      "logps/chosen": -228.59085083007812,
      "logps/rejected": -240.2413787841797,
      "loss": 0.3355,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6128778457641602,
      "rewards/margins": 3.4234275817871094,
      "rewards/rejected": -5.036304950714111,
      "step": 1211
    },
    {
      "epoch": 1.59,
      "learning_rate": 4.7364314339914716e-05,
      "logits/chosen": -2.223177671432495,
      "logits/rejected": -2.280726671218872,
      "logps/chosen": -187.680419921875,
      "logps/rejected": -279.1700134277344,
      "loss": 0.0374,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6807892322540283,
      "rewards/margins": 6.459383010864258,
      "rewards/rejected": -7.140172004699707,
      "step": 1212
    },
    {
      "epoch": 1.59,
      "learning_rate": 4.735965706355221e-05,
      "logits/chosen": -2.3972856998443604,
      "logits/rejected": -2.450835704803467,
      "logps/chosen": -180.9072723388672,
      "logps/rejected": -224.91830444335938,
      "loss": 0.2092,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4030086994171143,
      "rewards/margins": 3.2653090953826904,
      "rewards/rejected": -4.668317794799805,
      "step": 1213
    },
    {
      "epoch": 1.59,
      "learning_rate": 4.735499590548775e-05,
      "logits/chosen": -2.483901023864746,
      "logits/rejected": -2.579958915710449,
      "logps/chosen": -165.31178283691406,
      "logps/rejected": -212.93484497070312,
      "loss": 0.3618,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7332736253738403,
      "rewards/margins": 2.2384800910949707,
      "rewards/rejected": -3.9717535972595215,
      "step": 1214
    },
    {
      "epoch": 1.59,
      "learning_rate": 4.7350330866530536e-05,
      "logits/chosen": -2.4746031761169434,
      "logits/rejected": -2.5858569145202637,
      "logps/chosen": -234.17544555664062,
      "logps/rejected": -243.3843231201172,
      "loss": 0.1585,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7856451272964478,
      "rewards/margins": 3.430792808532715,
      "rewards/rejected": -5.216437816619873,
      "step": 1215
    },
    {
      "epoch": 1.59,
      "learning_rate": 4.734566194749043e-05,
      "logits/chosen": -2.1635468006134033,
      "logits/rejected": -2.184338092803955,
      "logps/chosen": -167.8646240234375,
      "logps/rejected": -198.2296142578125,
      "loss": 0.136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4915691614151,
      "rewards/margins": 3.2404394149780273,
      "rewards/rejected": -4.732008934020996,
      "step": 1216
    },
    {
      "epoch": 1.59,
      "learning_rate": 4.7340989149177955e-05,
      "logits/chosen": -2.4515089988708496,
      "logits/rejected": -2.4606142044067383,
      "logps/chosen": -184.1483154296875,
      "logps/rejected": -231.7574462890625,
      "loss": 0.1365,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9022963047027588,
      "rewards/margins": 4.141842842102051,
      "rewards/rejected": -6.0441389083862305,
      "step": 1217
    },
    {
      "epoch": 1.59,
      "learning_rate": 4.7336312472404345e-05,
      "logits/chosen": -2.495224714279175,
      "logits/rejected": -2.6189403533935547,
      "logps/chosen": -187.76014709472656,
      "logps/rejected": -287.38653564453125,
      "loss": 0.1484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5860087871551514,
      "rewards/margins": 4.361400604248047,
      "rewards/rejected": -5.947409629821777,
      "step": 1218
    },
    {
      "epoch": 1.6,
      "learning_rate": 4.733163191798147e-05,
      "logits/chosen": -2.3041515350341797,
      "logits/rejected": -2.3410122394561768,
      "logps/chosen": -182.3394775390625,
      "logps/rejected": -240.0974884033203,
      "loss": 0.2967,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4822897911071777,
      "rewards/margins": 3.956850290298462,
      "rewards/rejected": -5.4391398429870605,
      "step": 1219
    },
    {
      "epoch": 1.6,
      "learning_rate": 4.7326947486721894e-05,
      "logits/chosen": -2.5003819465637207,
      "logits/rejected": -2.486321449279785,
      "logps/chosen": -175.6728057861328,
      "logps/rejected": -249.07659912109375,
      "loss": 0.1647,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0928356647491455,
      "rewards/margins": 4.23591423034668,
      "rewards/rejected": -5.328750133514404,
      "step": 1220
    },
    {
      "epoch": 1.6,
      "learning_rate": 4.732225917943884e-05,
      "logits/chosen": -2.34039306640625,
      "logits/rejected": -2.2606441974639893,
      "logps/chosen": -209.9175262451172,
      "logps/rejected": -277.741455078125,
      "loss": 0.167,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1915336847305298,
      "rewards/margins": 4.357934474945068,
      "rewards/rejected": -5.54946756362915,
      "step": 1221
    },
    {
      "epoch": 1.6,
      "learning_rate": 4.7317566996946235e-05,
      "logits/chosen": -2.5158519744873047,
      "logits/rejected": -2.391768455505371,
      "logps/chosen": -282.4315490722656,
      "logps/rejected": -324.89337158203125,
      "loss": 0.0819,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4069547653198242,
      "rewards/margins": 4.744081497192383,
      "rewards/rejected": -6.151035785675049,
      "step": 1222
    },
    {
      "epoch": 1.6,
      "learning_rate": 4.731287094005863e-05,
      "logits/chosen": -2.2263362407684326,
      "logits/rejected": -2.276297092437744,
      "logps/chosen": -216.80142211914062,
      "logps/rejected": -220.71160888671875,
      "loss": 0.2046,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.122260093688965,
      "rewards/margins": 3.1513662338256836,
      "rewards/rejected": -5.27362585067749,
      "step": 1223
    },
    {
      "epoch": 1.6,
      "learning_rate": 4.730817100959128e-05,
      "logits/chosen": -2.3766252994537354,
      "logits/rejected": -2.406181812286377,
      "logps/chosen": -199.9800567626953,
      "logps/rejected": -254.28375244140625,
      "loss": 0.2671,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7991282939910889,
      "rewards/margins": 3.4454705715179443,
      "rewards/rejected": -5.244598388671875,
      "step": 1224
    },
    {
      "epoch": 1.6,
      "learning_rate": 4.730346720636011e-05,
      "logits/chosen": -2.5629076957702637,
      "logits/rejected": -2.5792136192321777,
      "logps/chosen": -177.7575225830078,
      "logps/rejected": -288.53857421875,
      "loss": 0.0896,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.251127004623413,
      "rewards/margins": 4.980374813079834,
      "rewards/rejected": -6.231501579284668,
      "step": 1225
    },
    {
      "epoch": 1.6,
      "learning_rate": 4.7298759531181717e-05,
      "logits/chosen": -2.423495054244995,
      "logits/rejected": -2.53666353225708,
      "logps/chosen": -225.38986206054688,
      "logps/rejected": -255.51467895507812,
      "loss": 0.0392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.030442476272583,
      "rewards/margins": 4.617352485656738,
      "rewards/rejected": -6.647795677185059,
      "step": 1226
    },
    {
      "epoch": 1.61,
      "learning_rate": 4.729404798487337e-05,
      "logits/chosen": -2.4218320846557617,
      "logits/rejected": -2.541698455810547,
      "logps/chosen": -189.56942749023438,
      "logps/rejected": -243.72561645507812,
      "loss": 0.0691,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4716410636901855,
      "rewards/margins": 4.515095233917236,
      "rewards/rejected": -5.98673677444458,
      "step": 1227
    },
    {
      "epoch": 1.61,
      "learning_rate": 4.7289332568252994e-05,
      "logits/chosen": -2.4882519245147705,
      "logits/rejected": -2.520144462585449,
      "logps/chosen": -194.36102294921875,
      "logps/rejected": -236.93948364257812,
      "loss": 0.2329,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6610629558563232,
      "rewards/margins": 3.4544930458068848,
      "rewards/rejected": -5.115556716918945,
      "step": 1228
    },
    {
      "epoch": 1.61,
      "learning_rate": 4.728461328213921e-05,
      "logits/chosen": -2.3431355953216553,
      "logits/rejected": -2.4315452575683594,
      "logps/chosen": -184.54681396484375,
      "logps/rejected": -237.89874267578125,
      "loss": 0.1567,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.615098237991333,
      "rewards/margins": 4.014577388763428,
      "rewards/rejected": -5.629674911499023,
      "step": 1229
    },
    {
      "epoch": 1.61,
      "learning_rate": 4.727989012735129e-05,
      "logits/chosen": -2.3083677291870117,
      "logits/rejected": -2.3745315074920654,
      "logps/chosen": -230.08078002929688,
      "logps/rejected": -273.61346435546875,
      "loss": 0.1073,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6126986742019653,
      "rewards/margins": 4.383516311645508,
      "rewards/rejected": -5.9962158203125,
      "step": 1230
    },
    {
      "epoch": 1.61,
      "learning_rate": 4.72751631047092e-05,
      "logits/chosen": -2.5195953845977783,
      "logits/rejected": -2.4491376876831055,
      "logps/chosen": -195.26785278320312,
      "logps/rejected": -225.06417846679688,
      "loss": 0.2724,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1910969018936157,
      "rewards/margins": 2.997663736343384,
      "rewards/rejected": -4.188760757446289,
      "step": 1231
    },
    {
      "epoch": 1.61,
      "learning_rate": 4.727043221503355e-05,
      "logits/chosen": -2.375887632369995,
      "logits/rejected": -2.3649067878723145,
      "logps/chosen": -190.8800811767578,
      "logps/rejected": -264.25982666015625,
      "loss": 0.1837,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8358839750289917,
      "rewards/margins": 4.462348461151123,
      "rewards/rejected": -5.298232078552246,
      "step": 1232
    },
    {
      "epoch": 1.61,
      "learning_rate": 4.7265697459145656e-05,
      "logits/chosen": -2.2352352142333984,
      "logits/rejected": -2.3413479328155518,
      "logps/chosen": -167.3771514892578,
      "logps/rejected": -217.39353942871094,
      "loss": 0.3485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4793394804000854,
      "rewards/margins": 2.465003490447998,
      "rewards/rejected": -3.944342851638794,
      "step": 1233
    },
    {
      "epoch": 1.61,
      "learning_rate": 4.7260958837867474e-05,
      "logits/chosen": -2.3950436115264893,
      "logits/rejected": -2.3509271144866943,
      "logps/chosen": -221.51156616210938,
      "logps/rejected": -274.8773193359375,
      "loss": 0.237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2772961854934692,
      "rewards/margins": 3.8298304080963135,
      "rewards/rejected": -5.107126712799072,
      "step": 1234
    },
    {
      "epoch": 1.62,
      "learning_rate": 4.725621635202164e-05,
      "logits/chosen": -2.217377185821533,
      "logits/rejected": -2.2314724922180176,
      "logps/chosen": -218.805419921875,
      "logps/rejected": -250.1678466796875,
      "loss": 0.274,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7175061702728271,
      "rewards/margins": 3.0603437423706055,
      "rewards/rejected": -4.777850151062012,
      "step": 1235
    },
    {
      "epoch": 1.62,
      "learning_rate": 4.7251470002431463e-05,
      "logits/chosen": -2.3143558502197266,
      "logits/rejected": -2.3055665493011475,
      "logps/chosen": -209.69020080566406,
      "logps/rejected": -267.2593994140625,
      "loss": 0.0749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4336559772491455,
      "rewards/margins": 4.277906894683838,
      "rewards/rejected": -5.711562633514404,
      "step": 1236
    },
    {
      "epoch": 1.62,
      "learning_rate": 4.724671978992093e-05,
      "logits/chosen": -2.5497217178344727,
      "logits/rejected": -2.357327699661255,
      "logps/chosen": -242.28077697753906,
      "logps/rejected": -254.4888916015625,
      "loss": 0.287,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.279671311378479,
      "rewards/margins": 4.20197057723999,
      "rewards/rejected": -5.48164176940918,
      "step": 1237
    },
    {
      "epoch": 1.62,
      "learning_rate": 4.724196571531469e-05,
      "logits/chosen": -2.209345579147339,
      "logits/rejected": -2.303694725036621,
      "logps/chosen": -213.60232543945312,
      "logps/rejected": -218.8167724609375,
      "loss": 0.3236,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.613269329071045,
      "rewards/margins": 2.6522085666656494,
      "rewards/rejected": -4.265478134155273,
      "step": 1238
    },
    {
      "epoch": 1.62,
      "learning_rate": 4.7237207779438066e-05,
      "logits/chosen": -2.289630889892578,
      "logits/rejected": -2.408109188079834,
      "logps/chosen": -220.02053833007812,
      "logps/rejected": -287.1589660644531,
      "loss": 0.1712,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9712376594543457,
      "rewards/margins": 5.0107197761535645,
      "rewards/rejected": -6.98195743560791,
      "step": 1239
    },
    {
      "epoch": 1.62,
      "learning_rate": 4.7232445983117045e-05,
      "logits/chosen": -2.221944808959961,
      "logits/rejected": -2.331416606903076,
      "logps/chosen": -204.5431365966797,
      "logps/rejected": -249.51193237304688,
      "loss": 0.2558,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8107631206512451,
      "rewards/margins": 2.60237455368042,
      "rewards/rejected": -4.413137435913086,
      "step": 1240
    },
    {
      "epoch": 1.62,
      "learning_rate": 4.7227680327178294e-05,
      "logits/chosen": -2.1551241874694824,
      "logits/rejected": -2.1501760482788086,
      "logps/chosen": -209.62078857421875,
      "logps/rejected": -237.73171997070312,
      "loss": 0.1494,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4279321432113647,
      "rewards/margins": 3.582655429840088,
      "rewards/rejected": -5.010587692260742,
      "step": 1241
    },
    {
      "epoch": 1.63,
      "learning_rate": 4.7222910812449126e-05,
      "logits/chosen": -2.5197300910949707,
      "logits/rejected": -2.501941442489624,
      "logps/chosen": -224.577880859375,
      "logps/rejected": -275.0819091796875,
      "loss": 0.1024,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8110631704330444,
      "rewards/margins": 4.743729591369629,
      "rewards/rejected": -5.554792404174805,
      "step": 1242
    },
    {
      "epoch": 1.63,
      "learning_rate": 4.7218137439757574e-05,
      "logits/chosen": -2.5621564388275146,
      "logits/rejected": -2.604523181915283,
      "logps/chosen": -160.7113037109375,
      "logps/rejected": -195.0877227783203,
      "loss": 0.2681,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9945410490036011,
      "rewards/margins": 2.9356656074523926,
      "rewards/rejected": -3.930206298828125,
      "step": 1243
    },
    {
      "epoch": 1.63,
      "learning_rate": 4.7213360209932286e-05,
      "logits/chosen": -2.444904088973999,
      "logits/rejected": -2.7083749771118164,
      "logps/chosen": -293.1217041015625,
      "logps/rejected": -356.0822448730469,
      "loss": 0.1383,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.486177921295166,
      "rewards/margins": 3.4033234119415283,
      "rewards/rejected": -4.889501571655273,
      "step": 1244
    },
    {
      "epoch": 1.63,
      "learning_rate": 4.720857912380261e-05,
      "logits/chosen": -2.3962252140045166,
      "logits/rejected": -2.4494524002075195,
      "logps/chosen": -245.69485473632812,
      "logps/rejected": -278.7045593261719,
      "loss": 0.1499,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5282708406448364,
      "rewards/margins": 3.444108486175537,
      "rewards/rejected": -4.972379207611084,
      "step": 1245
    },
    {
      "epoch": 1.63,
      "learning_rate": 4.7203794182198556e-05,
      "logits/chosen": -2.1511898040771484,
      "logits/rejected": -1.9930884838104248,
      "logps/chosen": -233.6335906982422,
      "logps/rejected": -218.43870544433594,
      "loss": 0.2441,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1958537101745605,
      "rewards/margins": 3.0618109703063965,
      "rewards/rejected": -4.257665157318115,
      "step": 1246
    },
    {
      "epoch": 1.63,
      "learning_rate": 4.7199005385950813e-05,
      "logits/chosen": -2.366896629333496,
      "logits/rejected": -2.437415361404419,
      "logps/chosen": -172.52511596679688,
      "logps/rejected": -218.31173706054688,
      "loss": 0.2482,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1920199394226074,
      "rewards/margins": 3.408511161804199,
      "rewards/rejected": -4.600531101226807,
      "step": 1247
    },
    {
      "epoch": 1.63,
      "learning_rate": 4.719421273589071e-05,
      "logits/chosen": -2.3530538082122803,
      "logits/rejected": -2.39542555809021,
      "logps/chosen": -214.77020263671875,
      "logps/rejected": -258.3968811035156,
      "loss": 0.2237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9770315289497375,
      "rewards/margins": 3.545395851135254,
      "rewards/rejected": -4.522427558898926,
      "step": 1248
    },
    {
      "epoch": 1.63,
      "learning_rate": 4.718941623285028e-05,
      "logits/chosen": -2.5741071701049805,
      "logits/rejected": -2.619910478591919,
      "logps/chosen": -190.63352966308594,
      "logps/rejected": -251.1927947998047,
      "loss": 0.135,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2246308326721191,
      "rewards/margins": 3.43021559715271,
      "rewards/rejected": -4.654846668243408,
      "step": 1249
    },
    {
      "epoch": 1.64,
      "learning_rate": 4.718461587766221e-05,
      "logits/chosen": -2.167393922805786,
      "logits/rejected": -2.1985065937042236,
      "logps/chosen": -184.55860900878906,
      "logps/rejected": -208.03286743164062,
      "loss": 0.2788,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9492772221565247,
      "rewards/margins": 3.1372551918029785,
      "rewards/rejected": -4.086532115936279,
      "step": 1250
    },
    {
      "epoch": 1.64,
      "learning_rate": 4.717981167115986e-05,
      "logits/chosen": -2.3610846996307373,
      "logits/rejected": -2.4981765747070312,
      "logps/chosen": -181.03359985351562,
      "logps/rejected": -254.43211364746094,
      "loss": 0.1609,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5473884344100952,
      "rewards/margins": 3.8110976219177246,
      "rewards/rejected": -5.358486175537109,
      "step": 1251
    },
    {
      "epoch": 1.64,
      "learning_rate": 4.717500361417724e-05,
      "logits/chosen": -2.58233904838562,
      "logits/rejected": -2.4361298084259033,
      "logps/chosen": -227.0497589111328,
      "logps/rejected": -225.55099487304688,
      "loss": 0.2847,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3530316352844238,
      "rewards/margins": 3.4566879272460938,
      "rewards/rejected": -4.809719562530518,
      "step": 1252
    },
    {
      "epoch": 1.64,
      "learning_rate": 4.717019170754905e-05,
      "logits/chosen": -2.4212703704833984,
      "logits/rejected": -2.353156805038452,
      "logps/chosen": -244.638671875,
      "logps/rejected": -234.03988647460938,
      "loss": 0.1988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9548088908195496,
      "rewards/margins": 3.1238605976104736,
      "rewards/rejected": -4.07866907119751,
      "step": 1253
    },
    {
      "epoch": 1.64,
      "learning_rate": 4.716537595211065e-05,
      "logits/chosen": -1.9767769575119019,
      "logits/rejected": -2.0549771785736084,
      "logps/chosen": -250.92616271972656,
      "logps/rejected": -285.457763671875,
      "loss": 0.1845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.187235951423645,
      "rewards/margins": 3.0077717304229736,
      "rewards/rejected": -4.195007801055908,
      "step": 1254
    },
    {
      "epoch": 1.64,
      "learning_rate": 4.716055634869807e-05,
      "logits/chosen": -2.5653319358825684,
      "logits/rejected": -2.4723970890045166,
      "logps/chosen": -192.06153869628906,
      "logps/rejected": -212.7266387939453,
      "loss": 0.2448,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9501335024833679,
      "rewards/margins": 3.1784751415252686,
      "rewards/rejected": -4.128608703613281,
      "step": 1255
    },
    {
      "epoch": 1.64,
      "learning_rate": 4.715573289814801e-05,
      "logits/chosen": -2.546515941619873,
      "logits/rejected": -2.493617057800293,
      "logps/chosen": -171.47616577148438,
      "logps/rejected": -203.3109130859375,
      "loss": 0.2335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9509350061416626,
      "rewards/margins": 2.794473648071289,
      "rewards/rejected": -3.745408535003662,
      "step": 1256
    },
    {
      "epoch": 1.65,
      "learning_rate": 4.715090560129782e-05,
      "logits/chosen": -1.9311320781707764,
      "logits/rejected": -1.997302770614624,
      "logps/chosen": -173.90911865234375,
      "logps/rejected": -195.12417602539062,
      "loss": 0.2128,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.22751878201961517,
      "rewards/margins": 3.4993607997894287,
      "rewards/rejected": -3.726879835128784,
      "step": 1257
    },
    {
      "epoch": 1.65,
      "learning_rate": 4.714607445898556e-05,
      "logits/chosen": -2.3790111541748047,
      "logits/rejected": -2.4061365127563477,
      "logps/chosen": -193.8656463623047,
      "logps/rejected": -248.78428649902344,
      "loss": 0.1781,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7878875732421875,
      "rewards/margins": 3.8841519355773926,
      "rewards/rejected": -4.67203950881958,
      "step": 1258
    },
    {
      "epoch": 1.65,
      "learning_rate": 4.714123947204991e-05,
      "logits/chosen": -2.5352847576141357,
      "logits/rejected": -2.5170440673828125,
      "logps/chosen": -207.12197875976562,
      "logps/rejected": -240.71539306640625,
      "loss": 0.1627,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1929060220718384,
      "rewards/margins": 3.0479984283447266,
      "rewards/rejected": -4.240904331207275,
      "step": 1259
    },
    {
      "epoch": 1.65,
      "learning_rate": 4.713640064133025e-05,
      "logits/chosen": -2.321063756942749,
      "logits/rejected": -2.323094367980957,
      "logps/chosen": -178.62144470214844,
      "logps/rejected": -220.08468627929688,
      "loss": 0.1978,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8668509721755981,
      "rewards/margins": 3.2052500247955322,
      "rewards/rejected": -4.07210111618042,
      "step": 1260
    },
    {
      "epoch": 1.65,
      "learning_rate": 4.7131557967666604e-05,
      "logits/chosen": -2.3726282119750977,
      "logits/rejected": -2.4751479625701904,
      "logps/chosen": -178.8557891845703,
      "logps/rejected": -251.33251953125,
      "loss": 0.1386,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6575889587402344,
      "rewards/margins": 3.7119858264923096,
      "rewards/rejected": -4.369575023651123,
      "step": 1261
    },
    {
      "epoch": 1.65,
      "learning_rate": 4.712671145189968e-05,
      "logits/chosen": -2.479706048965454,
      "logits/rejected": -2.519768238067627,
      "logps/chosen": -240.58731079101562,
      "logps/rejected": -287.4944763183594,
      "loss": 0.1786,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6783007979393005,
      "rewards/margins": 3.953369140625,
      "rewards/rejected": -4.631669998168945,
      "step": 1262
    },
    {
      "epoch": 1.65,
      "learning_rate": 4.712186109487085e-05,
      "logits/chosen": -2.2034120559692383,
      "logits/rejected": -2.2748920917510986,
      "logps/chosen": -176.4800262451172,
      "logps/rejected": -242.2554931640625,
      "loss": 0.0759,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.833525538444519,
      "rewards/margins": 4.219998836517334,
      "rewards/rejected": -5.053524494171143,
      "step": 1263
    },
    {
      "epoch": 1.65,
      "learning_rate": 4.7117006897422144e-05,
      "logits/chosen": -2.3310108184814453,
      "logits/rejected": -2.3973095417022705,
      "logps/chosen": -159.26260375976562,
      "logps/rejected": -178.69801330566406,
      "loss": 0.2582,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5862424969673157,
      "rewards/margins": 3.4215569496154785,
      "rewards/rejected": -4.00779914855957,
      "step": 1264
    },
    {
      "epoch": 1.66,
      "learning_rate": 4.7112148860396265e-05,
      "logits/chosen": -2.289217710494995,
      "logits/rejected": -2.3254284858703613,
      "logps/chosen": -198.0190887451172,
      "logps/rejected": -251.11077880859375,
      "loss": 0.1246,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1426129341125488,
      "rewards/margins": 4.011805534362793,
      "rewards/rejected": -5.154417991638184,
      "step": 1265
    },
    {
      "epoch": 1.66,
      "learning_rate": 4.7107286984636584e-05,
      "logits/chosen": -2.2878618240356445,
      "logits/rejected": -2.4422900676727295,
      "logps/chosen": -184.45816040039062,
      "logps/rejected": -227.22039794921875,
      "loss": 0.2988,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1406644582748413,
      "rewards/margins": 2.195034980773926,
      "rewards/rejected": -3.3356995582580566,
      "step": 1266
    },
    {
      "epoch": 1.66,
      "learning_rate": 4.710242127098714e-05,
      "logits/chosen": -2.411558151245117,
      "logits/rejected": -2.433676242828369,
      "logps/chosen": -276.55462646484375,
      "logps/rejected": -287.45538330078125,
      "loss": 0.1436,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3652961254119873,
      "rewards/margins": 2.717362403869629,
      "rewards/rejected": -4.082658290863037,
      "step": 1267
    },
    {
      "epoch": 1.66,
      "learning_rate": 4.7097551720292634e-05,
      "logits/chosen": -2.2663257122039795,
      "logits/rejected": -2.3046867847442627,
      "logps/chosen": -229.05191040039062,
      "logps/rejected": -234.79849243164062,
      "loss": 0.1054,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1178110837936401,
      "rewards/margins": 3.994016408920288,
      "rewards/rejected": -5.1118268966674805,
      "step": 1268
    },
    {
      "epoch": 1.66,
      "learning_rate": 4.709267833339843e-05,
      "logits/chosen": -2.4231812953948975,
      "logits/rejected": -2.46999192237854,
      "logps/chosen": -243.0467529296875,
      "logps/rejected": -288.1591796875,
      "loss": 0.0957,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6723252534866333,
      "rewards/margins": 4.546743869781494,
      "rewards/rejected": -5.219069480895996,
      "step": 1269
    },
    {
      "epoch": 1.66,
      "learning_rate": 4.708780111115057e-05,
      "logits/chosen": -2.2736947536468506,
      "logits/rejected": -2.318422794342041,
      "logps/chosen": -240.78549194335938,
      "logps/rejected": -262.1117858886719,
      "loss": 0.0696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0388414859771729,
      "rewards/margins": 3.992526054382324,
      "rewards/rejected": -5.031367301940918,
      "step": 1270
    },
    {
      "epoch": 1.66,
      "learning_rate": 4.708292005439575e-05,
      "logits/chosen": -2.3022494316101074,
      "logits/rejected": -2.331955671310425,
      "logps/chosen": -235.0003204345703,
      "logps/rejected": -286.7228698730469,
      "loss": 0.0884,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9852278232574463,
      "rewards/margins": 4.658226013183594,
      "rewards/rejected": -5.643453598022461,
      "step": 1271
    },
    {
      "epoch": 1.66,
      "learning_rate": 4.707803516398134e-05,
      "logits/chosen": -2.302123546600342,
      "logits/rejected": -2.2593071460723877,
      "logps/chosen": -219.12557983398438,
      "logps/rejected": -229.34735107421875,
      "loss": 0.2727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6084837913513184,
      "rewards/margins": 2.6438004970550537,
      "rewards/rejected": -4.252284049987793,
      "step": 1272
    },
    {
      "epoch": 1.67,
      "learning_rate": 4.707314644075536e-05,
      "logits/chosen": -2.2804455757141113,
      "logits/rejected": -2.313739538192749,
      "logps/chosen": -177.1968994140625,
      "logps/rejected": -253.0790557861328,
      "loss": 0.1938,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1521004438400269,
      "rewards/margins": 3.550403118133545,
      "rewards/rejected": -4.702503204345703,
      "step": 1273
    },
    {
      "epoch": 1.67,
      "learning_rate": 4.706825388556652e-05,
      "logits/chosen": -2.3808209896087646,
      "logits/rejected": -2.509162187576294,
      "logps/chosen": -252.71810913085938,
      "logps/rejected": -296.3233642578125,
      "loss": 0.1607,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4979945421218872,
      "rewards/margins": 3.630544424057007,
      "rewards/rejected": -5.128539085388184,
      "step": 1274
    },
    {
      "epoch": 1.67,
      "learning_rate": 4.706335749926417e-05,
      "logits/chosen": -2.3603124618530273,
      "logits/rejected": -2.442885637283325,
      "logps/chosen": -187.13800048828125,
      "logps/rejected": -226.97061157226562,
      "loss": 0.1575,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0760753154754639,
      "rewards/margins": 3.3448987007141113,
      "rewards/rejected": -4.420973777770996,
      "step": 1275
    },
    {
      "epoch": 1.67,
      "learning_rate": 4.7058457282698336e-05,
      "logits/chosen": -2.178051233291626,
      "logits/rejected": -2.302097797393799,
      "logps/chosen": -181.60252380371094,
      "logps/rejected": -298.5541687011719,
      "loss": 0.0962,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3809150457382202,
      "rewards/margins": 3.7876832485198975,
      "rewards/rejected": -5.168598175048828,
      "step": 1276
    },
    {
      "epoch": 1.67,
      "learning_rate": 4.7053553236719726e-05,
      "logits/chosen": -2.2287707328796387,
      "logits/rejected": -2.3011856079101562,
      "logps/chosen": -193.0646514892578,
      "logps/rejected": -230.15853881835938,
      "loss": 0.1986,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.612187385559082,
      "rewards/margins": 2.7522740364074707,
      "rewards/rejected": -4.3644609451293945,
      "step": 1277
    },
    {
      "epoch": 1.67,
      "learning_rate": 4.7048645362179686e-05,
      "logits/chosen": -2.3724539279937744,
      "logits/rejected": -2.3160030841827393,
      "logps/chosen": -199.37181091308594,
      "logps/rejected": -222.05191040039062,
      "loss": 0.1415,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6102732419967651,
      "rewards/margins": 3.988173723220825,
      "rewards/rejected": -4.598446369171143,
      "step": 1278
    },
    {
      "epoch": 1.67,
      "learning_rate": 4.7043733659930235e-05,
      "logits/chosen": -1.9961585998535156,
      "logits/rejected": -2.158682346343994,
      "logps/chosen": -148.25836181640625,
      "logps/rejected": -226.0478515625,
      "loss": 0.1392,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.005221962928772,
      "rewards/margins": 3.8446872234344482,
      "rewards/rejected": -4.84990930557251,
      "step": 1279
    },
    {
      "epoch": 1.68,
      "learning_rate": 4.703881813082406e-05,
      "logits/chosen": -2.171950101852417,
      "logits/rejected": -2.174027442932129,
      "logps/chosen": -188.34934997558594,
      "logps/rejected": -226.32675170898438,
      "loss": 0.2559,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7046524286270142,
      "rewards/margins": 3.046712875366211,
      "rewards/rejected": -4.751364707946777,
      "step": 1280
    },
    {
      "epoch": 1.68,
      "learning_rate": 4.703389877571451e-05,
      "logits/chosen": -2.3705697059631348,
      "logits/rejected": -2.3761351108551025,
      "logps/chosen": -262.95513916015625,
      "logps/rejected": -301.3341369628906,
      "loss": 0.3501,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5746833086013794,
      "rewards/margins": 3.5883994102478027,
      "rewards/rejected": -5.163082599639893,
      "step": 1281
    },
    {
      "epoch": 1.68,
      "learning_rate": 4.7028975595455615e-05,
      "logits/chosen": -2.2629284858703613,
      "logits/rejected": -2.3212735652923584,
      "logps/chosen": -148.93194580078125,
      "logps/rejected": -209.88656616210938,
      "loss": 0.2053,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2451274394989014,
      "rewards/margins": 3.617295026779175,
      "rewards/rejected": -4.862421989440918,
      "step": 1282
    },
    {
      "epoch": 1.68,
      "learning_rate": 4.7024048590902036e-05,
      "logits/chosen": -2.1855812072753906,
      "logits/rejected": -2.266613721847534,
      "logps/chosen": -208.891845703125,
      "logps/rejected": -263.1065979003906,
      "loss": 0.2646,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.979274868965149,
      "rewards/margins": 2.8052868843078613,
      "rewards/rejected": -4.784561634063721,
      "step": 1283
    },
    {
      "epoch": 1.68,
      "learning_rate": 4.701911776290912e-05,
      "logits/chosen": -2.373356342315674,
      "logits/rejected": -2.4201443195343018,
      "logps/chosen": -213.91403198242188,
      "logps/rejected": -248.12171936035156,
      "loss": 0.1366,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.634600281715393,
      "rewards/margins": 3.5006790161132812,
      "rewards/rejected": -5.135279655456543,
      "step": 1284
    },
    {
      "epoch": 1.68,
      "learning_rate": 4.7014183112332874e-05,
      "logits/chosen": -2.0015125274658203,
      "logits/rejected": -2.0453169345855713,
      "logps/chosen": -200.25445556640625,
      "logps/rejected": -244.46153259277344,
      "loss": 0.1716,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2220110893249512,
      "rewards/margins": 4.0147528648376465,
      "rewards/rejected": -5.236764430999756,
      "step": 1285
    },
    {
      "epoch": 1.68,
      "learning_rate": 4.700924464002998e-05,
      "logits/chosen": -2.3927388191223145,
      "logits/rejected": -2.5242536067962646,
      "logps/chosen": -209.62437438964844,
      "logps/rejected": -284.5310974121094,
      "loss": 0.1046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.037900447845459,
      "rewards/margins": 4.573543071746826,
      "rewards/rejected": -5.611443996429443,
      "step": 1286
    },
    {
      "epoch": 1.68,
      "learning_rate": 4.700430234685774e-05,
      "logits/chosen": -1.9511386156082153,
      "logits/rejected": -1.9067704677581787,
      "logps/chosen": -237.9818115234375,
      "logps/rejected": -313.5265197753906,
      "loss": 0.2084,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6042085886001587,
      "rewards/margins": 3.7971808910369873,
      "rewards/rejected": -5.401390075683594,
      "step": 1287
    },
    {
      "epoch": 1.69,
      "learning_rate": 4.699935623367419e-05,
      "logits/chosen": -2.0798680782318115,
      "logits/rejected": -2.097235679626465,
      "logps/chosen": -164.66754150390625,
      "logps/rejected": -223.6951904296875,
      "loss": 0.1752,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4378409385681152,
      "rewards/margins": 3.772213935852051,
      "rewards/rejected": -5.210054874420166,
      "step": 1288
    },
    {
      "epoch": 1.69,
      "learning_rate": 4.699440630133794e-05,
      "logits/chosen": -1.9853588342666626,
      "logits/rejected": -2.1096606254577637,
      "logps/chosen": -170.85031127929688,
      "logps/rejected": -239.245361328125,
      "loss": 0.153,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4343587160110474,
      "rewards/margins": 4.021684646606445,
      "rewards/rejected": -5.456043720245361,
      "step": 1289
    },
    {
      "epoch": 1.69,
      "learning_rate": 4.698945255070837e-05,
      "logits/chosen": -2.266202449798584,
      "logits/rejected": -2.3734381198883057,
      "logps/chosen": -241.38861083984375,
      "logps/rejected": -287.3223876953125,
      "loss": 0.1812,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.899042010307312,
      "rewards/margins": 4.466583728790283,
      "rewards/rejected": -6.365625381469727,
      "step": 1290
    },
    {
      "epoch": 1.69,
      "learning_rate": 4.698449498264543e-05,
      "logits/chosen": -2.0678675174713135,
      "logits/rejected": -2.1580851078033447,
      "logps/chosen": -177.24830627441406,
      "logps/rejected": -204.01907348632812,
      "loss": 0.2167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0937800407409668,
      "rewards/margins": 3.3943605422973633,
      "rewards/rejected": -4.48814058303833,
      "step": 1291
    },
    {
      "epoch": 1.69,
      "learning_rate": 4.697953359800977e-05,
      "logits/chosen": -2.3612008094787598,
      "logits/rejected": -2.470273017883301,
      "logps/chosen": -190.984375,
      "logps/rejected": -306.0478515625,
      "loss": 0.0471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2556102275848389,
      "rewards/margins": 5.612930774688721,
      "rewards/rejected": -6.868541240692139,
      "step": 1292
    },
    {
      "epoch": 1.69,
      "learning_rate": 4.697456839766271e-05,
      "logits/chosen": -2.3923254013061523,
      "logits/rejected": -2.461078643798828,
      "logps/chosen": -164.07113647460938,
      "logps/rejected": -214.48974609375,
      "loss": 0.1813,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.171428918838501,
      "rewards/margins": 2.888441562652588,
      "rewards/rejected": -4.059870719909668,
      "step": 1293
    },
    {
      "epoch": 1.69,
      "learning_rate": 4.6969599382466224e-05,
      "logits/chosen": -1.8895044326782227,
      "logits/rejected": -1.9640676975250244,
      "logps/chosen": -289.2847595214844,
      "logps/rejected": -292.143310546875,
      "loss": 0.2552,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2626152038574219,
      "rewards/margins": 3.0650792121887207,
      "rewards/rejected": -4.327694416046143,
      "step": 1294
    },
    {
      "epoch": 1.69,
      "learning_rate": 4.696462655328294e-05,
      "logits/chosen": -2.2127761840820312,
      "logits/rejected": -2.1274161338806152,
      "logps/chosen": -225.32937622070312,
      "logps/rejected": -221.54954528808594,
      "loss": 0.2618,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.374513864517212,
      "rewards/margins": 2.8716228008270264,
      "rewards/rejected": -4.246136665344238,
      "step": 1295
    },
    {
      "epoch": 1.7,
      "learning_rate": 4.695964991097616e-05,
      "logits/chosen": -2.295297861099243,
      "logits/rejected": -2.401890754699707,
      "logps/chosen": -183.36146545410156,
      "logps/rejected": -258.7024230957031,
      "loss": 0.0562,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.309377670288086,
      "rewards/margins": 4.095472812652588,
      "rewards/rejected": -5.404850959777832,
      "step": 1296
    },
    {
      "epoch": 1.7,
      "learning_rate": 4.695466945640985e-05,
      "logits/chosen": -2.3360753059387207,
      "logits/rejected": -2.359584093093872,
      "logps/chosen": -300.7992858886719,
      "logps/rejected": -348.9504089355469,
      "loss": 0.1097,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1483001708984375,
      "rewards/margins": 5.027990818023682,
      "rewards/rejected": -6.176291465759277,
      "step": 1297
    },
    {
      "epoch": 1.7,
      "learning_rate": 4.694968519044862e-05,
      "logits/chosen": -2.1906495094299316,
      "logits/rejected": -2.2294445037841797,
      "logps/chosen": -199.27716064453125,
      "logps/rejected": -267.9615173339844,
      "loss": 0.1167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9273327589035034,
      "rewards/margins": 4.393675804138184,
      "rewards/rejected": -5.321008682250977,
      "step": 1298
    },
    {
      "epoch": 1.7,
      "learning_rate": 4.6944697113957756e-05,
      "logits/chosen": -2.2078731060028076,
      "logits/rejected": -2.256173849105835,
      "logps/chosen": -191.87767028808594,
      "logps/rejected": -235.9379425048828,
      "loss": 0.229,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.396527886390686,
      "rewards/margins": 2.8328018188476562,
      "rewards/rejected": -4.229329586029053,
      "step": 1299
    },
    {
      "epoch": 1.7,
      "learning_rate": 4.69397052278032e-05,
      "logits/chosen": -2.1057164669036865,
      "logits/rejected": -2.2075157165527344,
      "logps/chosen": -159.22825622558594,
      "logps/rejected": -249.6995849609375,
      "loss": 0.2449,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.563463568687439,
      "rewards/margins": 3.6448473930358887,
      "rewards/rejected": -5.208311080932617,
      "step": 1300
    },
    {
      "epoch": 1.7,
      "learning_rate": 4.6934709532851576e-05,
      "logits/chosen": -1.976876974105835,
      "logits/rejected": -2.1168007850646973,
      "logps/chosen": -190.38943481445312,
      "logps/rejected": -240.00660705566406,
      "loss": 0.1593,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6499911546707153,
      "rewards/margins": 4.014796257019043,
      "rewards/rejected": -5.664787769317627,
      "step": 1301
    },
    {
      "epoch": 1.7,
      "learning_rate": 4.692971002997013e-05,
      "logits/chosen": -2.2851758003234863,
      "logits/rejected": -2.3062691688537598,
      "logps/chosen": -213.82559204101562,
      "logps/rejected": -243.2498779296875,
      "loss": 0.2007,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8377220630645752,
      "rewards/margins": 3.4279751777648926,
      "rewards/rejected": -5.265697002410889,
      "step": 1302
    },
    {
      "epoch": 1.71,
      "learning_rate": 4.69247067200268e-05,
      "logits/chosen": -2.130112886428833,
      "logits/rejected": -2.21236515045166,
      "logps/chosen": -232.77862548828125,
      "logps/rejected": -242.76791381835938,
      "loss": 0.1511,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.000922679901123,
      "rewards/margins": 3.678673028945923,
      "rewards/rejected": -5.679595947265625,
      "step": 1303
    },
    {
      "epoch": 1.71,
      "learning_rate": 4.691969960389017e-05,
      "logits/chosen": -2.2007036209106445,
      "logits/rejected": -2.316322088241577,
      "logps/chosen": -195.4522247314453,
      "logps/rejected": -319.2470397949219,
      "loss": 0.0627,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5212204456329346,
      "rewards/margins": 5.476848602294922,
      "rewards/rejected": -6.998068809509277,
      "step": 1304
    },
    {
      "epoch": 1.71,
      "learning_rate": 4.6914688682429496e-05,
      "logits/chosen": -2.1540913581848145,
      "logits/rejected": -2.2840263843536377,
      "logps/chosen": -164.23712158203125,
      "logps/rejected": -267.3701477050781,
      "loss": 0.154,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7769471406936646,
      "rewards/margins": 4.149184226989746,
      "rewards/rejected": -5.926130771636963,
      "step": 1305
    },
    {
      "epoch": 1.71,
      "learning_rate": 4.690967395651469e-05,
      "logits/chosen": -2.2483348846435547,
      "logits/rejected": -2.29646372795105,
      "logps/chosen": -198.18312072753906,
      "logps/rejected": -250.00401306152344,
      "loss": 0.059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.107490301132202,
      "rewards/margins": 4.931715965270996,
      "rewards/rejected": -7.039206027984619,
      "step": 1306
    },
    {
      "epoch": 1.71,
      "learning_rate": 4.6904655427016314e-05,
      "logits/chosen": -2.344970703125,
      "logits/rejected": -2.3008577823638916,
      "logps/chosen": -242.4059600830078,
      "logps/rejected": -261.99755859375,
      "loss": 0.211,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.909515380859375,
      "rewards/margins": 4.080948829650879,
      "rewards/rejected": -5.990464687347412,
      "step": 1307
    },
    {
      "epoch": 1.71,
      "learning_rate": 4.6899633094805604e-05,
      "logits/chosen": -2.0280845165252686,
      "logits/rejected": -2.0966038703918457,
      "logps/chosen": -218.1485595703125,
      "logps/rejected": -247.9756317138672,
      "loss": 0.1563,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.803059697151184,
      "rewards/margins": 3.7059831619262695,
      "rewards/rejected": -5.509043216705322,
      "step": 1308
    },
    {
      "epoch": 1.71,
      "learning_rate": 4.689460696075446e-05,
      "logits/chosen": -2.481982707977295,
      "logits/rejected": -2.4297902584075928,
      "logps/chosen": -226.631103515625,
      "logps/rejected": -258.3005676269531,
      "loss": 0.1634,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8385543823242188,
      "rewards/margins": 3.6741104125976562,
      "rewards/rejected": -5.512665271759033,
      "step": 1309
    },
    {
      "epoch": 1.71,
      "learning_rate": 4.688957702573542e-05,
      "logits/chosen": -2.086306095123291,
      "logits/rejected": -2.0635809898376465,
      "logps/chosen": -227.90188598632812,
      "logps/rejected": -282.12567138671875,
      "loss": 0.2416,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.798666000366211,
      "rewards/margins": 3.5320932865142822,
      "rewards/rejected": -5.330759048461914,
      "step": 1310
    },
    {
      "epoch": 1.72,
      "learning_rate": 4.6884543290621706e-05,
      "logits/chosen": -2.2293214797973633,
      "logits/rejected": -2.256197452545166,
      "logps/chosen": -173.7129669189453,
      "logps/rejected": -233.62586975097656,
      "loss": 0.4036,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.9006210565567017,
      "rewards/margins": 2.86419677734375,
      "rewards/rejected": -4.764817714691162,
      "step": 1311
    },
    {
      "epoch": 1.72,
      "learning_rate": 4.687950575628718e-05,
      "logits/chosen": -2.299515962600708,
      "logits/rejected": -2.3766961097717285,
      "logps/chosen": -179.56732177734375,
      "logps/rejected": -205.10609436035156,
      "loss": 0.2016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3757731914520264,
      "rewards/margins": 2.9517035484313965,
      "rewards/rejected": -4.327476501464844,
      "step": 1312
    },
    {
      "epoch": 1.72,
      "learning_rate": 4.6874464423606376e-05,
      "logits/chosen": -2.057006597518921,
      "logits/rejected": -2.160207748413086,
      "logps/chosen": -161.31712341308594,
      "logps/rejected": -270.31756591796875,
      "loss": 0.2467,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.198062539100647,
      "rewards/margins": 3.796602725982666,
      "rewards/rejected": -4.994665145874023,
      "step": 1313
    },
    {
      "epoch": 1.72,
      "learning_rate": 4.68694192934545e-05,
      "logits/chosen": -2.2372817993164062,
      "logits/rejected": -2.2993078231811523,
      "logps/chosen": -181.70199584960938,
      "logps/rejected": -230.60086059570312,
      "loss": 0.1501,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5110223293304443,
      "rewards/margins": 3.5902843475341797,
      "rewards/rejected": -5.101306438446045,
      "step": 1314
    },
    {
      "epoch": 1.72,
      "learning_rate": 4.6864370366707366e-05,
      "logits/chosen": -2.2221570014953613,
      "logits/rejected": -2.290891170501709,
      "logps/chosen": -180.98440551757812,
      "logps/rejected": -241.5121612548828,
      "loss": 0.208,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7390170097351074,
      "rewards/margins": 3.494724988937378,
      "rewards/rejected": -5.2337422370910645,
      "step": 1315
    },
    {
      "epoch": 1.72,
      "learning_rate": 4.685931764424152e-05,
      "logits/chosen": -2.3075151443481445,
      "logits/rejected": -2.3091671466827393,
      "logps/chosen": -141.8212890625,
      "logps/rejected": -196.77894592285156,
      "loss": 0.161,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0228283405303955,
      "rewards/margins": 3.695592164993286,
      "rewards/rejected": -4.718420505523682,
      "step": 1316
    },
    {
      "epoch": 1.72,
      "learning_rate": 4.6854261126934104e-05,
      "logits/chosen": -2.3127944469451904,
      "logits/rejected": -2.3393845558166504,
      "logps/chosen": -210.4659423828125,
      "logps/rejected": -244.98114013671875,
      "loss": 0.1695,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8345630168914795,
      "rewards/margins": 3.397213935852051,
      "rewards/rejected": -5.231777191162109,
      "step": 1317
    },
    {
      "epoch": 1.72,
      "learning_rate": 4.684920081566295e-05,
      "logits/chosen": -2.4269180297851562,
      "logits/rejected": -2.3492565155029297,
      "logps/chosen": -233.5777130126953,
      "logps/rejected": -288.9328918457031,
      "loss": 0.2221,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7820842266082764,
      "rewards/margins": 4.016000270843506,
      "rewards/rejected": -5.7980852127075195,
      "step": 1318
    },
    {
      "epoch": 1.73,
      "learning_rate": 4.684413671130655e-05,
      "logits/chosen": -2.320209503173828,
      "logits/rejected": -2.377079486846924,
      "logps/chosen": -207.27413940429688,
      "logps/rejected": -261.8699645996094,
      "loss": 0.1259,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6501234769821167,
      "rewards/margins": 4.435462474822998,
      "rewards/rejected": -6.085585594177246,
      "step": 1319
    },
    {
      "epoch": 1.73,
      "learning_rate": 4.683906881474405e-05,
      "logits/chosen": -2.298753499984741,
      "logits/rejected": -2.3262522220611572,
      "logps/chosen": -184.51959228515625,
      "logps/rejected": -243.18218994140625,
      "loss": 0.0632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8446322679519653,
      "rewards/margins": 3.7827131748199463,
      "rewards/rejected": -5.627345085144043,
      "step": 1320
    },
    {
      "epoch": 1.73,
      "learning_rate": 4.6833997126855236e-05,
      "logits/chosen": -2.3990066051483154,
      "logits/rejected": -2.450573444366455,
      "logps/chosen": -223.92630004882812,
      "logps/rejected": -278.1924133300781,
      "loss": 0.2113,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.5321887731552124,
      "rewards/margins": 3.711320638656616,
      "rewards/rejected": -5.243509292602539,
      "step": 1321
    },
    {
      "epoch": 1.73,
      "learning_rate": 4.682892164852058e-05,
      "logits/chosen": -2.2027530670166016,
      "logits/rejected": -2.214686632156372,
      "logps/chosen": -239.34274291992188,
      "logps/rejected": -282.7929992675781,
      "loss": 0.2324,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4780960083007812,
      "rewards/margins": 3.398163318634033,
      "rewards/rejected": -4.8762593269348145,
      "step": 1322
    },
    {
      "epoch": 1.73,
      "learning_rate": 4.68238423806212e-05,
      "logits/chosen": -2.098435640335083,
      "logits/rejected": -2.160342216491699,
      "logps/chosen": -175.86923217773438,
      "logps/rejected": -210.9672393798828,
      "loss": 0.2131,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5760440826416016,
      "rewards/margins": 2.946066379547119,
      "rewards/rejected": -4.522110462188721,
      "step": 1323
    },
    {
      "epoch": 1.73,
      "learning_rate": 4.681875932403885e-05,
      "logits/chosen": -2.0780820846557617,
      "logits/rejected": -2.2287166118621826,
      "logps/chosen": -220.93051147460938,
      "logps/rejected": -261.7267761230469,
      "loss": 0.1903,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6689494848251343,
      "rewards/margins": 3.8627724647521973,
      "rewards/rejected": -5.531722068786621,
      "step": 1324
    },
    {
      "epoch": 1.73,
      "learning_rate": 4.6813672479656e-05,
      "logits/chosen": -2.196786642074585,
      "logits/rejected": -2.3375864028930664,
      "logps/chosen": -191.28599548339844,
      "logps/rejected": -281.8190612792969,
      "loss": 0.1382,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.868288278579712,
      "rewards/margins": 4.053330898284912,
      "rewards/rejected": -5.921619415283203,
      "step": 1325
    },
    {
      "epoch": 1.74,
      "learning_rate": 4.680858184835572e-05,
      "logits/chosen": -2.0055956840515137,
      "logits/rejected": -2.090541124343872,
      "logps/chosen": -181.52813720703125,
      "logps/rejected": -213.05496215820312,
      "loss": 0.171,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.124434232711792,
      "rewards/margins": 3.510894536972046,
      "rewards/rejected": -4.635329246520996,
      "step": 1326
    },
    {
      "epoch": 1.74,
      "learning_rate": 4.6803487431021766e-05,
      "logits/chosen": -2.1428966522216797,
      "logits/rejected": -2.2327733039855957,
      "logps/chosen": -174.168701171875,
      "logps/rejected": -196.87881469726562,
      "loss": 0.222,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8680024147033691,
      "rewards/margins": 3.048375129699707,
      "rewards/rejected": -4.916378021240234,
      "step": 1327
    },
    {
      "epoch": 1.74,
      "learning_rate": 4.679838922853853e-05,
      "logits/chosen": -2.3784139156341553,
      "logits/rejected": -2.481027126312256,
      "logps/chosen": -230.0206298828125,
      "logps/rejected": -294.965087890625,
      "loss": 0.168,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9025079011917114,
      "rewards/margins": 4.027909278869629,
      "rewards/rejected": -5.930417537689209,
      "step": 1328
    },
    {
      "epoch": 1.74,
      "learning_rate": 4.679328724179109e-05,
      "logits/chosen": -2.269122362136841,
      "logits/rejected": -2.279000997543335,
      "logps/chosen": -191.70877075195312,
      "logps/rejected": -215.52420043945312,
      "loss": 0.1554,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.063866138458252,
      "rewards/margins": 3.1374106407165527,
      "rewards/rejected": -5.201276779174805,
      "step": 1329
    },
    {
      "epoch": 1.74,
      "learning_rate": 4.6788181471665155e-05,
      "logits/chosen": -1.8508931398391724,
      "logits/rejected": -1.8643546104431152,
      "logps/chosen": -205.4655303955078,
      "logps/rejected": -203.88031005859375,
      "loss": 0.1438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.508162021636963,
      "rewards/margins": 3.4267380237579346,
      "rewards/rejected": -4.934900283813477,
      "step": 1330
    },
    {
      "epoch": 1.74,
      "learning_rate": 4.67830719190471e-05,
      "logits/chosen": -2.4449105262756348,
      "logits/rejected": -2.512434720993042,
      "logps/chosen": -239.13856506347656,
      "logps/rejected": -255.4873046875,
      "loss": 0.1123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7588633298873901,
      "rewards/margins": 4.075693607330322,
      "rewards/rejected": -5.83455753326416,
      "step": 1331
    },
    {
      "epoch": 1.74,
      "learning_rate": 4.677795858482398e-05,
      "logits/chosen": -2.2655084133148193,
      "logits/rejected": -2.2468154430389404,
      "logps/chosen": -166.35696411132812,
      "logps/rejected": -241.24954223632812,
      "loss": 0.1517,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7497742176055908,
      "rewards/margins": 3.5795230865478516,
      "rewards/rejected": -5.3292975425720215,
      "step": 1332
    },
    {
      "epoch": 1.74,
      "learning_rate": 4.677284146988346e-05,
      "logits/chosen": -2.377413749694824,
      "logits/rejected": -2.4802167415618896,
      "logps/chosen": -257.7947998046875,
      "logps/rejected": -343.67376708984375,
      "loss": 0.0402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.074228048324585,
      "rewards/margins": 5.048233985900879,
      "rewards/rejected": -7.122462272644043,
      "step": 1333
    },
    {
      "epoch": 1.75,
      "learning_rate": 4.67677205751139e-05,
      "logits/chosen": -2.3548061847686768,
      "logits/rejected": -2.4688026905059814,
      "logps/chosen": -202.11593627929688,
      "logps/rejected": -286.65057373046875,
      "loss": 0.1916,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.193711519241333,
      "rewards/margins": 5.112841606140137,
      "rewards/rejected": -6.306553840637207,
      "step": 1334
    },
    {
      "epoch": 1.75,
      "learning_rate": 4.67625959014043e-05,
      "logits/chosen": -2.31046199798584,
      "logits/rejected": -2.319591760635376,
      "logps/chosen": -231.41473388671875,
      "logps/rejected": -231.21859741210938,
      "loss": 0.1506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5231584310531616,
      "rewards/margins": 3.534045934677124,
      "rewards/rejected": -5.057204723358154,
      "step": 1335
    },
    {
      "epoch": 1.75,
      "learning_rate": 4.675746744964432e-05,
      "logits/chosen": -2.524745464324951,
      "logits/rejected": -2.5354037284851074,
      "logps/chosen": -280.01513671875,
      "logps/rejected": -341.6401672363281,
      "loss": 0.1042,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.0203945636749268,
      "rewards/margins": 4.112698554992676,
      "rewards/rejected": -6.133092880249023,
      "step": 1336
    },
    {
      "epoch": 1.75,
      "learning_rate": 4.675233522072426e-05,
      "logits/chosen": -2.4641785621643066,
      "logits/rejected": -2.4546477794647217,
      "logps/chosen": -204.9468536376953,
      "logps/rejected": -250.145263671875,
      "loss": 0.1739,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4312738180160522,
      "rewards/margins": 3.9612386226654053,
      "rewards/rejected": -5.392512798309326,
      "step": 1337
    },
    {
      "epoch": 1.75,
      "learning_rate": 4.674719921553511e-05,
      "logits/chosen": -2.3155698776245117,
      "logits/rejected": -2.3693156242370605,
      "logps/chosen": -194.86477661132812,
      "logps/rejected": -220.62965393066406,
      "loss": 0.1797,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.0034408569335938,
      "rewards/margins": 3.511406898498535,
      "rewards/rejected": -5.514848709106445,
      "step": 1338
    },
    {
      "epoch": 1.75,
      "learning_rate": 4.6742059434968476e-05,
      "logits/chosen": -1.9812606573104858,
      "logits/rejected": -2.0393075942993164,
      "logps/chosen": -197.7154541015625,
      "logps/rejected": -251.43502807617188,
      "loss": 0.2036,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.391548991203308,
      "rewards/margins": 3.891415596008301,
      "rewards/rejected": -5.282964706420898,
      "step": 1339
    },
    {
      "epoch": 1.75,
      "learning_rate": 4.673691587991667e-05,
      "logits/chosen": -2.2613768577575684,
      "logits/rejected": -2.2857589721679688,
      "logps/chosen": -211.1699981689453,
      "logps/rejected": -271.5691833496094,
      "loss": 0.3857,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.122715950012207,
      "rewards/margins": 3.556459903717041,
      "rewards/rejected": -5.679176330566406,
      "step": 1340
    },
    {
      "epoch": 1.75,
      "learning_rate": 4.673176855127258e-05,
      "logits/chosen": -2.3039798736572266,
      "logits/rejected": -2.286431312561035,
      "logps/chosen": -209.6785430908203,
      "logps/rejected": -263.2655334472656,
      "loss": 0.182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8972321152687073,
      "rewards/margins": 4.346026420593262,
      "rewards/rejected": -5.243258953094482,
      "step": 1341
    },
    {
      "epoch": 1.76,
      "learning_rate": 4.672661744992984e-05,
      "logits/chosen": -2.2901010513305664,
      "logits/rejected": -2.3650054931640625,
      "logps/chosen": -195.44326782226562,
      "logps/rejected": -268.2812805175781,
      "loss": 0.1694,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.497087001800537,
      "rewards/margins": 4.011203289031982,
      "rewards/rejected": -5.5082902908325195,
      "step": 1342
    },
    {
      "epoch": 1.76,
      "learning_rate": 4.672146257678268e-05,
      "logits/chosen": -2.4022762775421143,
      "logits/rejected": -2.3658864498138428,
      "logps/chosen": -255.15890502929688,
      "logps/rejected": -328.7245788574219,
      "loss": 0.1034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.900804877281189,
      "rewards/margins": 4.233809471130371,
      "rewards/rejected": -6.13461446762085,
      "step": 1343
    },
    {
      "epoch": 1.76,
      "learning_rate": 4.6716303932726004e-05,
      "logits/chosen": -2.4683516025543213,
      "logits/rejected": -2.4238386154174805,
      "logps/chosen": -181.5071563720703,
      "logps/rejected": -225.39805603027344,
      "loss": 0.2542,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.721781849861145,
      "rewards/margins": 3.4932799339294434,
      "rewards/rejected": -5.215061664581299,
      "step": 1344
    },
    {
      "epoch": 1.76,
      "learning_rate": 4.671114151865536e-05,
      "logits/chosen": -2.38026762008667,
      "logits/rejected": -2.38236927986145,
      "logps/chosen": -227.99789428710938,
      "logps/rejected": -277.05975341796875,
      "loss": 0.1147,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6720448732376099,
      "rewards/margins": 4.081615447998047,
      "rewards/rejected": -5.753660202026367,
      "step": 1345
    },
    {
      "epoch": 1.76,
      "learning_rate": 4.6705975335466966e-05,
      "logits/chosen": -2.504228115081787,
      "logits/rejected": -2.5582098960876465,
      "logps/chosen": -272.26788330078125,
      "logps/rejected": -299.64056396484375,
      "loss": 0.144,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.06329083442688,
      "rewards/margins": 3.614590644836426,
      "rewards/rejected": -5.677881240844727,
      "step": 1346
    },
    {
      "epoch": 1.76,
      "learning_rate": 4.6700805384057687e-05,
      "logits/chosen": -2.207796096801758,
      "logits/rejected": -2.2440619468688965,
      "logps/chosen": -251.6385040283203,
      "logps/rejected": -267.8268127441406,
      "loss": 0.0849,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2752389907836914,
      "rewards/margins": 3.857135772705078,
      "rewards/rejected": -5.1323747634887695,
      "step": 1347
    },
    {
      "epoch": 1.76,
      "learning_rate": 4.669563166532503e-05,
      "logits/chosen": -2.1402127742767334,
      "logits/rejected": -2.2065505981445312,
      "logps/chosen": -188.67706298828125,
      "logps/rejected": -221.7375946044922,
      "loss": 0.1975,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9871059656143188,
      "rewards/margins": 2.8594772815704346,
      "rewards/rejected": -4.846583366394043,
      "step": 1348
    },
    {
      "epoch": 1.77,
      "learning_rate": 4.669045418016719e-05,
      "logits/chosen": -2.2698256969451904,
      "logits/rejected": -2.274237871170044,
      "logps/chosen": -197.71026611328125,
      "logps/rejected": -252.61614990234375,
      "loss": 0.1191,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6768132448196411,
      "rewards/margins": 3.3656845092773438,
      "rewards/rejected": -5.042497634887695,
      "step": 1349
    },
    {
      "epoch": 1.77,
      "learning_rate": 4.668527292948298e-05,
      "logits/chosen": -2.256514072418213,
      "logits/rejected": -2.34165358543396,
      "logps/chosen": -176.2838592529297,
      "logps/rejected": -236.6180877685547,
      "loss": 0.1925,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.650253176689148,
      "rewards/margins": 3.62697172164917,
      "rewards/rejected": -5.277225017547607,
      "step": 1350
    },
    {
      "epoch": 1.77,
      "learning_rate": 4.668008791417188e-05,
      "logits/chosen": -2.31141996383667,
      "logits/rejected": -2.3515350818634033,
      "logps/chosen": -238.41036987304688,
      "logps/rejected": -284.0400390625,
      "loss": 0.0897,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6227517127990723,
      "rewards/margins": 4.4259443283081055,
      "rewards/rejected": -6.048696041107178,
      "step": 1351
    },
    {
      "epoch": 1.77,
      "learning_rate": 4.6674899135134024e-05,
      "logits/chosen": -2.0742459297180176,
      "logits/rejected": -2.1106269359588623,
      "logps/chosen": -172.02809143066406,
      "logps/rejected": -193.6383514404297,
      "loss": 0.237,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9648047089576721,
      "rewards/margins": 2.872191905975342,
      "rewards/rejected": -3.836996555328369,
      "step": 1352
    },
    {
      "epoch": 1.77,
      "learning_rate": 4.666970659327019e-05,
      "logits/chosen": -1.8485875129699707,
      "logits/rejected": -1.9281960725784302,
      "logps/chosen": -219.42877197265625,
      "logps/rejected": -291.0449523925781,
      "loss": 0.1646,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.602189540863037,
      "rewards/margins": 3.566793441772461,
      "rewards/rejected": -5.168982982635498,
      "step": 1353
    },
    {
      "epoch": 1.77,
      "learning_rate": 4.666451028948183e-05,
      "logits/chosen": -2.3288824558258057,
      "logits/rejected": -2.427269458770752,
      "logps/chosen": -238.8002471923828,
      "logps/rejected": -297.3604431152344,
      "loss": 0.0734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7097375392913818,
      "rewards/margins": 4.595492362976074,
      "rewards/rejected": -6.305230140686035,
      "step": 1354
    },
    {
      "epoch": 1.77,
      "learning_rate": 4.665931022467105e-05,
      "logits/chosen": -2.096285104751587,
      "logits/rejected": -2.1877405643463135,
      "logps/chosen": -181.7442169189453,
      "logps/rejected": -265.5242614746094,
      "loss": 0.1665,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.873305320739746,
      "rewards/margins": 3.2641849517822266,
      "rewards/rejected": -5.137490272521973,
      "step": 1355
    },
    {
      "epoch": 1.77,
      "learning_rate": 4.665410639974057e-05,
      "logits/chosen": -2.1947834491729736,
      "logits/rejected": -2.359546422958374,
      "logps/chosen": -203.13490295410156,
      "logps/rejected": -254.12127685546875,
      "loss": 0.189,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1950178146362305,
      "rewards/margins": 4.417459487915039,
      "rewards/rejected": -5.612477779388428,
      "step": 1356
    },
    {
      "epoch": 1.78,
      "learning_rate": 4.664889881559381e-05,
      "logits/chosen": -2.353022336959839,
      "logits/rejected": -2.4439656734466553,
      "logps/chosen": -230.41729736328125,
      "logps/rejected": -242.9822998046875,
      "loss": 0.0859,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2911330461502075,
      "rewards/margins": 4.007154941558838,
      "rewards/rejected": -5.298287391662598,
      "step": 1357
    },
    {
      "epoch": 1.78,
      "learning_rate": 4.66436874731348e-05,
      "logits/chosen": -2.055748462677002,
      "logits/rejected": -2.069406509399414,
      "logps/chosen": -215.91287231445312,
      "logps/rejected": -224.975341796875,
      "loss": 0.1346,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.132076382637024,
      "rewards/margins": 3.4015748500823975,
      "rewards/rejected": -4.533651351928711,
      "step": 1358
    },
    {
      "epoch": 1.78,
      "learning_rate": 4.663847237326827e-05,
      "logits/chosen": -2.2017364501953125,
      "logits/rejected": -2.1791625022888184,
      "logps/chosen": -195.00912475585938,
      "logps/rejected": -187.93539428710938,
      "loss": 0.2966,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.539417028427124,
      "rewards/margins": 2.387049674987793,
      "rewards/rejected": -3.926466703414917,
      "step": 1359
    },
    {
      "epoch": 1.78,
      "learning_rate": 4.663325351689956e-05,
      "logits/chosen": -2.2987170219421387,
      "logits/rejected": -2.302344560623169,
      "logps/chosen": -186.64981079101562,
      "logps/rejected": -223.81967163085938,
      "loss": 0.182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5449962615966797,
      "rewards/margins": 3.964876413345337,
      "rewards/rejected": -5.509873390197754,
      "step": 1360
    },
    {
      "epoch": 1.78,
      "learning_rate": 4.662803090493469e-05,
      "logits/chosen": -2.1506714820861816,
      "logits/rejected": -2.18817400932312,
      "logps/chosen": -257.8224792480469,
      "logps/rejected": -328.1229248046875,
      "loss": 0.1701,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0192809104919434,
      "rewards/margins": 3.9228391647338867,
      "rewards/rejected": -5.94212007522583,
      "step": 1361
    },
    {
      "epoch": 1.78,
      "learning_rate": 4.6622804538280305e-05,
      "logits/chosen": -2.2823166847229004,
      "logits/rejected": -2.34694242477417,
      "logps/chosen": -195.22393798828125,
      "logps/rejected": -255.1166534423828,
      "loss": 0.1154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.038851261138916,
      "rewards/margins": 3.4105467796325684,
      "rewards/rejected": -5.449398040771484,
      "step": 1362
    },
    {
      "epoch": 1.78,
      "learning_rate": 4.661757441784373e-05,
      "logits/chosen": -2.2843942642211914,
      "logits/rejected": -2.220759630203247,
      "logps/chosen": -219.33355712890625,
      "logps/rejected": -290.8352966308594,
      "loss": 0.0939,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6054083108901978,
      "rewards/margins": 4.078475475311279,
      "rewards/rejected": -5.6838836669921875,
      "step": 1363
    },
    {
      "epoch": 1.79,
      "learning_rate": 4.6612340544532935e-05,
      "logits/chosen": -2.2316360473632812,
      "logits/rejected": -2.3092989921569824,
      "logps/chosen": -200.7064971923828,
      "logps/rejected": -254.74070739746094,
      "loss": 0.1406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5896083116531372,
      "rewards/margins": 3.663377285003662,
      "rewards/rejected": -5.252985954284668,
      "step": 1364
    },
    {
      "epoch": 1.79,
      "learning_rate": 4.660710291925652e-05,
      "logits/chosen": -2.365083694458008,
      "logits/rejected": -2.388625144958496,
      "logps/chosen": -230.10348510742188,
      "logps/rejected": -289.7381286621094,
      "loss": 0.1506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6961638927459717,
      "rewards/margins": 3.8952834606170654,
      "rewards/rejected": -5.591447830200195,
      "step": 1365
    },
    {
      "epoch": 1.79,
      "learning_rate": 4.660186154292375e-05,
      "logits/chosen": -2.2262802124023438,
      "logits/rejected": -2.279958486557007,
      "logps/chosen": -163.28163146972656,
      "logps/rejected": -202.38223266601562,
      "loss": 0.2373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6737639904022217,
      "rewards/margins": 3.0033376216888428,
      "rewards/rejected": -4.6771016120910645,
      "step": 1366
    },
    {
      "epoch": 1.79,
      "learning_rate": 4.659661641644456e-05,
      "logits/chosen": -2.447139263153076,
      "logits/rejected": -2.4368622303009033,
      "logps/chosen": -242.7307891845703,
      "logps/rejected": -283.7666320800781,
      "loss": 0.1133,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5830082893371582,
      "rewards/margins": 5.0759172439575195,
      "rewards/rejected": -6.6589250564575195,
      "step": 1367
    },
    {
      "epoch": 1.79,
      "learning_rate": 4.6591367540729515e-05,
      "logits/chosen": -2.524048328399658,
      "logits/rejected": -2.555225133895874,
      "logps/chosen": -243.4622039794922,
      "logps/rejected": -252.66217041015625,
      "loss": 0.1873,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.028730869293213,
      "rewards/margins": 3.0651321411132812,
      "rewards/rejected": -5.093863487243652,
      "step": 1368
    },
    {
      "epoch": 1.79,
      "learning_rate": 4.658611491668983e-05,
      "logits/chosen": -2.3555378913879395,
      "logits/rejected": -2.4342939853668213,
      "logps/chosen": -194.3604736328125,
      "logps/rejected": -248.84271240234375,
      "loss": 0.1389,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7463260889053345,
      "rewards/margins": 3.3878800868988037,
      "rewards/rejected": -5.1342058181762695,
      "step": 1369
    },
    {
      "epoch": 1.79,
      "learning_rate": 4.658085854523737e-05,
      "logits/chosen": -2.1884450912475586,
      "logits/rejected": -2.365642786026001,
      "logps/chosen": -185.33209228515625,
      "logps/rejected": -265.621337890625,
      "loss": 0.1218,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.218601942062378,
      "rewards/margins": 3.699212074279785,
      "rewards/rejected": -5.917815208435059,
      "step": 1370
    },
    {
      "epoch": 1.79,
      "learning_rate": 4.657559842728467e-05,
      "logits/chosen": -2.103135347366333,
      "logits/rejected": -2.149179697036743,
      "logps/chosen": -214.1287841796875,
      "logps/rejected": -276.39312744140625,
      "loss": 0.1939,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.805924654006958,
      "rewards/margins": 4.1208319664001465,
      "rewards/rejected": -5.926756381988525,
      "step": 1371
    },
    {
      "epoch": 1.8,
      "learning_rate": 4.657033456374489e-05,
      "logits/chosen": -1.8944284915924072,
      "logits/rejected": -1.9346532821655273,
      "logps/chosen": -216.1498565673828,
      "logps/rejected": -308.89453125,
      "loss": 0.109,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7429003715515137,
      "rewards/margins": 4.299877643585205,
      "rewards/rejected": -6.0427775382995605,
      "step": 1372
    },
    {
      "epoch": 1.8,
      "learning_rate": 4.6565066955531863e-05,
      "logits/chosen": -1.657898187637329,
      "logits/rejected": -1.6330841779708862,
      "logps/chosen": -213.9786834716797,
      "logps/rejected": -256.35113525390625,
      "loss": 0.3878,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.641648054122925,
      "rewards/margins": 2.252370834350586,
      "rewards/rejected": -4.89401912689209,
      "step": 1373
    },
    {
      "epoch": 1.8,
      "learning_rate": 4.655979560356005e-05,
      "logits/chosen": -2.202993154525757,
      "logits/rejected": -2.2177088260650635,
      "logps/chosen": -236.91598510742188,
      "logps/rejected": -295.3562927246094,
      "loss": 0.033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0502257347106934,
      "rewards/margins": 5.525638103485107,
      "rewards/rejected": -7.575864315032959,
      "step": 1374
    },
    {
      "epoch": 1.8,
      "learning_rate": 4.655452050874459e-05,
      "logits/chosen": -2.14239501953125,
      "logits/rejected": -2.1285078525543213,
      "logps/chosen": -187.77191162109375,
      "logps/rejected": -228.86451721191406,
      "loss": 0.214,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.5740537643432617,
      "rewards/margins": 3.127923011779785,
      "rewards/rejected": -5.701977252960205,
      "step": 1375
    },
    {
      "epoch": 1.8,
      "learning_rate": 4.654924167200123e-05,
      "logits/chosen": -2.3964757919311523,
      "logits/rejected": -2.476527214050293,
      "logps/chosen": -214.957763671875,
      "logps/rejected": -286.00726318359375,
      "loss": 0.1093,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0560764074325562,
      "rewards/margins": 5.730381965637207,
      "rewards/rejected": -6.786458969116211,
      "step": 1376
    },
    {
      "epoch": 1.8,
      "learning_rate": 4.6543959094246417e-05,
      "logits/chosen": -2.488255739212036,
      "logits/rejected": -2.6213529109954834,
      "logps/chosen": -249.02215576171875,
      "logps/rejected": -291.32379150390625,
      "loss": 0.1211,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5869184732437134,
      "rewards/margins": 4.215513706207275,
      "rewards/rejected": -5.802432060241699,
      "step": 1377
    },
    {
      "epoch": 1.8,
      "learning_rate": 4.653867277639721e-05,
      "logits/chosen": -2.311802387237549,
      "logits/rejected": -2.3098201751708984,
      "logps/chosen": -172.78297424316406,
      "logps/rejected": -222.96755981445312,
      "loss": 0.1179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1166205406188965,
      "rewards/margins": 3.5452730655670166,
      "rewards/rejected": -5.661893367767334,
      "step": 1378
    },
    {
      "epoch": 1.8,
      "learning_rate": 4.653338271937132e-05,
      "logits/chosen": -2.316065788269043,
      "logits/rejected": -2.3509740829467773,
      "logps/chosen": -202.41773986816406,
      "logps/rejected": -270.28985595703125,
      "loss": 0.1453,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8998184204101562,
      "rewards/margins": 3.892240524291992,
      "rewards/rejected": -5.79205846786499,
      "step": 1379
    },
    {
      "epoch": 1.81,
      "learning_rate": 4.6528088924087134e-05,
      "logits/chosen": -2.3619024753570557,
      "logits/rejected": -2.308654308319092,
      "logps/chosen": -263.7723083496094,
      "logps/rejected": -302.5215759277344,
      "loss": 0.2607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3372600078582764,
      "rewards/margins": 3.7634713649749756,
      "rewards/rejected": -6.100731372833252,
      "step": 1380
    },
    {
      "epoch": 1.81,
      "learning_rate": 4.652279139146366e-05,
      "logits/chosen": -2.067979097366333,
      "logits/rejected": -2.1202504634857178,
      "logps/chosen": -210.7864532470703,
      "logps/rejected": -255.50363159179688,
      "loss": 0.1086,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8815836906433105,
      "rewards/margins": 4.555849552154541,
      "rewards/rejected": -6.437433242797852,
      "step": 1381
    },
    {
      "epoch": 1.81,
      "learning_rate": 4.651749012242057e-05,
      "logits/chosen": -2.3216161727905273,
      "logits/rejected": -2.276050329208374,
      "logps/chosen": -210.85401916503906,
      "logps/rejected": -233.73196411132812,
      "loss": 0.311,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.473651885986328,
      "rewards/margins": 2.5943374633789062,
      "rewards/rejected": -5.067989349365234,
      "step": 1382
    },
    {
      "epoch": 1.81,
      "learning_rate": 4.6512185117878184e-05,
      "logits/chosen": -2.3193976879119873,
      "logits/rejected": -2.3813366889953613,
      "logps/chosen": -205.80455017089844,
      "logps/rejected": -243.294921875,
      "loss": 0.2155,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.865469217300415,
      "rewards/margins": 4.035120487213135,
      "rewards/rejected": -5.900589466094971,
      "step": 1383
    },
    {
      "epoch": 1.81,
      "learning_rate": 4.6506876378757445e-05,
      "logits/chosen": -2.170536994934082,
      "logits/rejected": -2.177908182144165,
      "logps/chosen": -169.406494140625,
      "logps/rejected": -262.20025634765625,
      "loss": 0.1339,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5915058851242065,
      "rewards/margins": 4.764673233032227,
      "rewards/rejected": -6.356178283691406,
      "step": 1384
    },
    {
      "epoch": 1.81,
      "learning_rate": 4.650156390598e-05,
      "logits/chosen": -1.956441879272461,
      "logits/rejected": -1.9708319902420044,
      "logps/chosen": -194.17105102539062,
      "logps/rejected": -242.59666442871094,
      "loss": 0.2926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0661697387695312,
      "rewards/margins": 3.1960580348968506,
      "rewards/rejected": -5.262228012084961,
      "step": 1385
    },
    {
      "epoch": 1.81,
      "learning_rate": 4.6496247700468064e-05,
      "logits/chosen": -2.279972553253174,
      "logits/rejected": -2.320099115371704,
      "logps/chosen": -180.3889617919922,
      "logps/rejected": -240.9708709716797,
      "loss": 0.1561,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7037296295166016,
      "rewards/margins": 3.612116575241089,
      "rewards/rejected": -5.3158464431762695,
      "step": 1386
    },
    {
      "epoch": 1.82,
      "learning_rate": 4.6490927763144586e-05,
      "logits/chosen": -2.3013417720794678,
      "logits/rejected": -2.2452809810638428,
      "logps/chosen": -198.49270629882812,
      "logps/rejected": -250.34814453125,
      "loss": 0.2868,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7538549900054932,
      "rewards/margins": 3.3283233642578125,
      "rewards/rejected": -5.082179069519043,
      "step": 1387
    },
    {
      "epoch": 1.82,
      "learning_rate": 4.6485604094933113e-05,
      "logits/chosen": -2.6034908294677734,
      "logits/rejected": -2.501837730407715,
      "logps/chosen": -241.82763671875,
      "logps/rejected": -259.59930419921875,
      "loss": 0.226,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8224003314971924,
      "rewards/margins": 4.394843101501465,
      "rewards/rejected": -6.217243671417236,
      "step": 1388
    },
    {
      "epoch": 1.82,
      "learning_rate": 4.648027669675784e-05,
      "logits/chosen": -1.8462001085281372,
      "logits/rejected": -1.890073537826538,
      "logps/chosen": -166.96856689453125,
      "logps/rejected": -255.78610229492188,
      "loss": 0.2828,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7682936191558838,
      "rewards/margins": 3.327087640762329,
      "rewards/rejected": -5.095381259918213,
      "step": 1389
    },
    {
      "epoch": 1.82,
      "learning_rate": 4.647494556954363e-05,
      "logits/chosen": -2.199425458908081,
      "logits/rejected": -2.260338306427002,
      "logps/chosen": -194.01919555664062,
      "logps/rejected": -243.82003784179688,
      "loss": 0.2729,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3266735076904297,
      "rewards/margins": 3.0939698219299316,
      "rewards/rejected": -5.420642852783203,
      "step": 1390
    },
    {
      "epoch": 1.82,
      "learning_rate": 4.646961071421597e-05,
      "logits/chosen": -2.479255199432373,
      "logits/rejected": -2.396368980407715,
      "logps/chosen": -178.45083618164062,
      "logps/rejected": -193.24459838867188,
      "loss": 0.1679,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2833914756774902,
      "rewards/margins": 2.904581308364868,
      "rewards/rejected": -4.187972545623779,
      "step": 1391
    },
    {
      "epoch": 1.82,
      "learning_rate": 4.646427213170102e-05,
      "logits/chosen": -2.217855453491211,
      "logits/rejected": -2.2055749893188477,
      "logps/chosen": -197.3143310546875,
      "logps/rejected": -273.6307067871094,
      "loss": 0.2277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.942773699760437,
      "rewards/margins": 3.781911849975586,
      "rewards/rejected": -4.7246856689453125,
      "step": 1392
    },
    {
      "epoch": 1.82,
      "learning_rate": 4.6458929822925566e-05,
      "logits/chosen": -2.133974552154541,
      "logits/rejected": -2.2947840690612793,
      "logps/chosen": -196.52053833007812,
      "logps/rejected": -273.97747802734375,
      "loss": 0.2357,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2982733249664307,
      "rewards/margins": 3.5180678367614746,
      "rewards/rejected": -4.816340923309326,
      "step": 1393
    },
    {
      "epoch": 1.82,
      "learning_rate": 4.645358378881704e-05,
      "logits/chosen": -2.507310152053833,
      "logits/rejected": -2.5766844749450684,
      "logps/chosen": -179.09832763671875,
      "logps/rejected": -204.78404235839844,
      "loss": 0.3219,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1929552555084229,
      "rewards/margins": 2.7457828521728516,
      "rewards/rejected": -3.9387383460998535,
      "step": 1394
    },
    {
      "epoch": 1.83,
      "learning_rate": 4.644823403030355e-05,
      "logits/chosen": -2.13411545753479,
      "logits/rejected": -2.2497849464416504,
      "logps/chosen": -193.55628967285156,
      "logps/rejected": -234.6313934326172,
      "loss": 0.1959,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2598943710327148,
      "rewards/margins": 3.595004081726074,
      "rewards/rejected": -4.854898452758789,
      "step": 1395
    },
    {
      "epoch": 1.83,
      "learning_rate": 4.644288054831381e-05,
      "logits/chosen": -2.157371997833252,
      "logits/rejected": -2.2184062004089355,
      "logps/chosen": -171.0150146484375,
      "logps/rejected": -226.55062866210938,
      "loss": 0.1302,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0181159973144531,
      "rewards/margins": 3.4827423095703125,
      "rewards/rejected": -4.500858306884766,
      "step": 1396
    },
    {
      "epoch": 1.83,
      "learning_rate": 4.643752334377721e-05,
      "logits/chosen": -2.3214855194091797,
      "logits/rejected": -2.367702007293701,
      "logps/chosen": -180.56222534179688,
      "logps/rejected": -226.59286499023438,
      "loss": 0.1739,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2419052124023438,
      "rewards/margins": 3.2632222175598145,
      "rewards/rejected": -4.505127429962158,
      "step": 1397
    },
    {
      "epoch": 1.83,
      "learning_rate": 4.643216241762377e-05,
      "logits/chosen": -2.4462032318115234,
      "logits/rejected": -2.4511361122131348,
      "logps/chosen": -193.83444213867188,
      "logps/rejected": -230.96469116210938,
      "loss": 0.2973,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.9721289873123169,
      "rewards/margins": 2.934811592102051,
      "rewards/rejected": -3.9069406986236572,
      "step": 1398
    },
    {
      "epoch": 1.83,
      "learning_rate": 4.642679777078417e-05,
      "logits/chosen": -2.4158241748809814,
      "logits/rejected": -2.453239679336548,
      "logps/chosen": -198.2194061279297,
      "logps/rejected": -246.531005859375,
      "loss": 0.0574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5831554532051086,
      "rewards/margins": 4.099462985992432,
      "rewards/rejected": -4.682618141174316,
      "step": 1399
    },
    {
      "epoch": 1.83,
      "learning_rate": 4.642142940418973e-05,
      "logits/chosen": -2.476083517074585,
      "logits/rejected": -2.496399402618408,
      "logps/chosen": -254.40286254882812,
      "logps/rejected": -267.8065490722656,
      "loss": 0.1294,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5694355964660645,
      "rewards/margins": 4.015815734863281,
      "rewards/rejected": -4.585251808166504,
      "step": 1400
    },
    {
      "epoch": 1.83,
      "learning_rate": 4.641605731877241e-05,
      "logits/chosen": -2.420027256011963,
      "logits/rejected": -2.445709466934204,
      "logps/chosen": -193.45823669433594,
      "logps/rejected": -229.07589721679688,
      "loss": 0.1181,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.91693115234375,
      "rewards/margins": 3.285738706588745,
      "rewards/rejected": -4.202670097351074,
      "step": 1401
    },
    {
      "epoch": 1.83,
      "learning_rate": 4.6410681515464815e-05,
      "logits/chosen": -2.453171491622925,
      "logits/rejected": -2.4817469120025635,
      "logps/chosen": -167.9488525390625,
      "logps/rejected": -211.94888305664062,
      "loss": 0.2742,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7758989334106445,
      "rewards/margins": 2.3340702056884766,
      "rewards/rejected": -3.109968900680542,
      "step": 1402
    },
    {
      "epoch": 1.84,
      "learning_rate": 4.640530199520021e-05,
      "logits/chosen": -2.478593587875366,
      "logits/rejected": -2.4444589614868164,
      "logps/chosen": -274.9038391113281,
      "logps/rejected": -317.6105041503906,
      "loss": 0.1598,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0752410888671875,
      "rewards/margins": 3.3734335899353027,
      "rewards/rejected": -4.44867467880249,
      "step": 1403
    },
    {
      "epoch": 1.84,
      "learning_rate": 4.639991875891248e-05,
      "logits/chosen": -1.9536941051483154,
      "logits/rejected": -2.0602893829345703,
      "logps/chosen": -164.46133422851562,
      "logps/rejected": -239.60308837890625,
      "loss": 0.1324,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1708040237426758,
      "rewards/margins": 3.239039659500122,
      "rewards/rejected": -4.409843444824219,
      "step": 1404
    },
    {
      "epoch": 1.84,
      "learning_rate": 4.639453180753619e-05,
      "logits/chosen": -2.305426597595215,
      "logits/rejected": -2.310289144515991,
      "logps/chosen": -179.86114501953125,
      "logps/rejected": -254.47314453125,
      "loss": 0.1235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4751981496810913,
      "rewards/margins": 4.430361270904541,
      "rewards/rejected": -4.905559539794922,
      "step": 1405
    },
    {
      "epoch": 1.84,
      "learning_rate": 4.638914114200652e-05,
      "logits/chosen": -1.8774938583374023,
      "logits/rejected": -1.9185031652450562,
      "logps/chosen": -173.3539276123047,
      "logps/rejected": -220.61851501464844,
      "loss": 0.2646,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2160310745239258,
      "rewards/margins": 2.826627016067505,
      "rewards/rejected": -4.042657852172852,
      "step": 1406
    },
    {
      "epoch": 1.84,
      "learning_rate": 4.638374676325931e-05,
      "logits/chosen": -2.312690496444702,
      "logits/rejected": -2.472609519958496,
      "logps/chosen": -178.3687744140625,
      "logps/rejected": -245.68804931640625,
      "loss": 0.1414,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2376123666763306,
      "rewards/margins": 3.6323118209838867,
      "rewards/rejected": -4.8699235916137695,
      "step": 1407
    },
    {
      "epoch": 1.84,
      "learning_rate": 4.637834867223102e-05,
      "logits/chosen": -2.4253156185150146,
      "logits/rejected": -2.5353803634643555,
      "logps/chosen": -192.8672332763672,
      "logps/rejected": -227.572265625,
      "loss": 0.1833,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8559720516204834,
      "rewards/margins": 3.2029502391815186,
      "rewards/rejected": -4.05892276763916,
      "step": 1408
    },
    {
      "epoch": 1.84,
      "learning_rate": 4.63729468698588e-05,
      "logits/chosen": -2.4336678981781006,
      "logits/rejected": -2.5701303482055664,
      "logps/chosen": -169.47442626953125,
      "logps/rejected": -245.16151428222656,
      "loss": 0.1092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9117591381072998,
      "rewards/margins": 3.5345542430877686,
      "rewards/rejected": -4.446313381195068,
      "step": 1409
    },
    {
      "epoch": 1.85,
      "learning_rate": 4.636754135708041e-05,
      "logits/chosen": -2.405302047729492,
      "logits/rejected": -2.4277071952819824,
      "logps/chosen": -194.17935180664062,
      "logps/rejected": -215.50399780273438,
      "loss": 0.2359,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0027927160263062,
      "rewards/margins": 3.0121240615844727,
      "rewards/rejected": -4.014916896820068,
      "step": 1410
    },
    {
      "epoch": 1.85,
      "learning_rate": 4.636213213483427e-05,
      "logits/chosen": -2.4292078018188477,
      "logits/rejected": -2.4207870960235596,
      "logps/chosen": -178.75831604003906,
      "logps/rejected": -193.9530029296875,
      "loss": 0.2236,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8789994716644287,
      "rewards/margins": 2.7491774559020996,
      "rewards/rejected": -3.62817645072937,
      "step": 1411
    },
    {
      "epoch": 1.85,
      "learning_rate": 4.635671920405942e-05,
      "logits/chosen": -2.354250192642212,
      "logits/rejected": -2.4390838146209717,
      "logps/chosen": -170.90542602539062,
      "logps/rejected": -211.04017639160156,
      "loss": 0.1892,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9057228565216064,
      "rewards/margins": 2.9422061443328857,
      "rewards/rejected": -3.847928762435913,
      "step": 1412
    },
    {
      "epoch": 1.85,
      "learning_rate": 4.635130256569558e-05,
      "logits/chosen": -2.297285795211792,
      "logits/rejected": -2.392590045928955,
      "logps/chosen": -168.90338134765625,
      "logps/rejected": -240.53675842285156,
      "loss": 0.1883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4901635646820068,
      "rewards/margins": 3.915353775024414,
      "rewards/rejected": -5.405517578125,
      "step": 1413
    },
    {
      "epoch": 1.85,
      "learning_rate": 4.634588222068307e-05,
      "logits/chosen": -2.1840977668762207,
      "logits/rejected": -2.1191656589508057,
      "logps/chosen": -253.16091918945312,
      "logps/rejected": -288.4150695800781,
      "loss": 0.2244,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.961954653263092,
      "rewards/margins": 3.244297504425049,
      "rewards/rejected": -4.206252098083496,
      "step": 1414
    },
    {
      "epoch": 1.85,
      "learning_rate": 4.63404581699629e-05,
      "logits/chosen": -2.4387269020080566,
      "logits/rejected": -2.4661765098571777,
      "logps/chosen": -205.93373107910156,
      "logps/rejected": -220.48907470703125,
      "loss": 0.3181,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2564948797225952,
      "rewards/margins": 2.7682459354400635,
      "rewards/rejected": -4.024740695953369,
      "step": 1415
    },
    {
      "epoch": 1.85,
      "learning_rate": 4.633503041447669e-05,
      "logits/chosen": -2.3245558738708496,
      "logits/rejected": -2.4340667724609375,
      "logps/chosen": -189.07936096191406,
      "logps/rejected": -247.6138916015625,
      "loss": 0.1197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9286269545555115,
      "rewards/margins": 4.146144390106201,
      "rewards/rejected": -5.074770927429199,
      "step": 1416
    },
    {
      "epoch": 1.85,
      "learning_rate": 4.632959895516672e-05,
      "logits/chosen": -2.2829172611236572,
      "logits/rejected": -2.447025775909424,
      "logps/chosen": -224.01718139648438,
      "logps/rejected": -285.6311950683594,
      "loss": 0.1772,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3857183456420898,
      "rewards/margins": 3.0930581092834473,
      "rewards/rejected": -4.478775978088379,
      "step": 1417
    },
    {
      "epoch": 1.86,
      "learning_rate": 4.6324163792975906e-05,
      "logits/chosen": -2.3150711059570312,
      "logits/rejected": -2.3638498783111572,
      "logps/chosen": -239.34747314453125,
      "logps/rejected": -272.7017822265625,
      "loss": 0.124,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.116297721862793,
      "rewards/margins": 4.121063232421875,
      "rewards/rejected": -5.237360954284668,
      "step": 1418
    },
    {
      "epoch": 1.86,
      "learning_rate": 4.63187249288478e-05,
      "logits/chosen": -2.2839043140411377,
      "logits/rejected": -2.2470085620880127,
      "logps/chosen": -180.84278869628906,
      "logps/rejected": -192.83535766601562,
      "loss": 0.2197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1662708520889282,
      "rewards/margins": 2.785388946533203,
      "rewards/rejected": -3.951659917831421,
      "step": 1419
    },
    {
      "epoch": 1.86,
      "learning_rate": 4.631328236372662e-05,
      "logits/chosen": -2.356921672821045,
      "logits/rejected": -2.308830738067627,
      "logps/chosen": -184.10110473632812,
      "logps/rejected": -209.7080078125,
      "loss": 0.2305,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.795234203338623,
      "rewards/margins": 2.5254967212677,
      "rewards/rejected": -4.320731163024902,
      "step": 1420
    },
    {
      "epoch": 1.86,
      "learning_rate": 4.630783609855719e-05,
      "logits/chosen": -1.8619661331176758,
      "logits/rejected": -1.8003392219543457,
      "logps/chosen": -173.55296325683594,
      "logps/rejected": -202.5664520263672,
      "loss": 0.2976,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6667727828025818,
      "rewards/margins": 3.061650037765503,
      "rewards/rejected": -3.7284226417541504,
      "step": 1421
    },
    {
      "epoch": 1.86,
      "learning_rate": 4.6302386134285026e-05,
      "logits/chosen": -2.434025287628174,
      "logits/rejected": -2.4339070320129395,
      "logps/chosen": -221.09487915039062,
      "logps/rejected": -241.01637268066406,
      "loss": 0.2679,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.9217256307601929,
      "rewards/margins": 2.6306025981903076,
      "rewards/rejected": -4.552328109741211,
      "step": 1422
    },
    {
      "epoch": 1.86,
      "learning_rate": 4.629693247185624e-05,
      "logits/chosen": -2.406184673309326,
      "logits/rejected": -2.3965976238250732,
      "logps/chosen": -233.80032348632812,
      "logps/rejected": -279.02288818359375,
      "loss": 0.2578,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3796286582946777,
      "rewards/margins": 3.7771825790405273,
      "rewards/rejected": -5.156811237335205,
      "step": 1423
    },
    {
      "epoch": 1.86,
      "learning_rate": 4.629147511221759e-05,
      "logits/chosen": -2.2653684616088867,
      "logits/rejected": -2.3729100227355957,
      "logps/chosen": -148.75238037109375,
      "logps/rejected": -179.93528747558594,
      "loss": 0.1934,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1495842933654785,
      "rewards/margins": 2.708228349685669,
      "rewards/rejected": -3.8578124046325684,
      "step": 1424
    },
    {
      "epoch": 1.86,
      "learning_rate": 4.628601405631652e-05,
      "logits/chosen": -2.2800092697143555,
      "logits/rejected": -2.31841778755188,
      "logps/chosen": -181.82470703125,
      "logps/rejected": -225.3800506591797,
      "loss": 0.1134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0458605289459229,
      "rewards/margins": 3.472771406173706,
      "rewards/rejected": -4.518631458282471,
      "step": 1425
    },
    {
      "epoch": 1.87,
      "learning_rate": 4.6280549305101063e-05,
      "logits/chosen": -2.3783209323883057,
      "logits/rejected": -2.4350523948669434,
      "logps/chosen": -259.91650390625,
      "logps/rejected": -312.69049072265625,
      "loss": 0.1717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.517119288444519,
      "rewards/margins": 3.184443950653076,
      "rewards/rejected": -4.701562881469727,
      "step": 1426
    },
    {
      "epoch": 1.87,
      "learning_rate": 4.6275080859519926e-05,
      "logits/chosen": -2.379199981689453,
      "logits/rejected": -2.4917795658111572,
      "logps/chosen": -213.16268920898438,
      "logps/rejected": -263.35296630859375,
      "loss": 0.1122,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0016521215438843,
      "rewards/margins": 4.081844806671143,
      "rewards/rejected": -5.08349609375,
      "step": 1427
    },
    {
      "epoch": 1.87,
      "learning_rate": 4.626960872052245e-05,
      "logits/chosen": -2.497894525527954,
      "logits/rejected": -2.427412509918213,
      "logps/chosen": -236.83901977539062,
      "logps/rejected": -263.41766357421875,
      "loss": 0.1137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0873581171035767,
      "rewards/margins": 3.7583518028259277,
      "rewards/rejected": -4.845709323883057,
      "step": 1428
    },
    {
      "epoch": 1.87,
      "learning_rate": 4.62641328890586e-05,
      "logits/chosen": -1.8372935056686401,
      "logits/rejected": -1.892336130142212,
      "logps/chosen": -197.54855346679688,
      "logps/rejected": -207.12667846679688,
      "loss": 0.2948,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3495253324508667,
      "rewards/margins": 2.6124796867370605,
      "rewards/rejected": -3.9620046615600586,
      "step": 1429
    },
    {
      "epoch": 1.87,
      "learning_rate": 4.625865336607901e-05,
      "logits/chosen": -2.2670607566833496,
      "logits/rejected": -2.344111680984497,
      "logps/chosen": -214.10784912109375,
      "logps/rejected": -266.7064208984375,
      "loss": 0.1006,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1706020832061768,
      "rewards/margins": 4.015186786651611,
      "rewards/rejected": -5.185789108276367,
      "step": 1430
    },
    {
      "epoch": 1.87,
      "learning_rate": 4.625317015253493e-05,
      "logits/chosen": -2.4320297241210938,
      "logits/rejected": -2.542137622833252,
      "logps/chosen": -179.33941650390625,
      "logps/rejected": -225.77798461914062,
      "loss": 0.1293,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0320031642913818,
      "rewards/margins": 3.152787685394287,
      "rewards/rejected": -4.184791088104248,
      "step": 1431
    },
    {
      "epoch": 1.87,
      "learning_rate": 4.624768324937827e-05,
      "logits/chosen": -1.8749370574951172,
      "logits/rejected": -1.8897876739501953,
      "logps/chosen": -208.49624633789062,
      "logps/rejected": -241.69107055664062,
      "loss": 0.258,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2621411085128784,
      "rewards/margins": 2.4016261100769043,
      "rewards/rejected": -3.663767099380493,
      "step": 1432
    },
    {
      "epoch": 1.88,
      "learning_rate": 4.624219265756158e-05,
      "logits/chosen": -2.201465606689453,
      "logits/rejected": -2.145068883895874,
      "logps/chosen": -205.04013061523438,
      "logps/rejected": -254.39373779296875,
      "loss": 0.113,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8891902565956116,
      "rewards/margins": 4.0929059982299805,
      "rewards/rejected": -4.9820966720581055,
      "step": 1433
    },
    {
      "epoch": 1.88,
      "learning_rate": 4.6236698378038026e-05,
      "logits/chosen": -2.4089853763580322,
      "logits/rejected": -2.4648375511169434,
      "logps/chosen": -199.67010498046875,
      "logps/rejected": -249.1591796875,
      "loss": 0.1373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.287391185760498,
      "rewards/margins": 4.044575214385986,
      "rewards/rejected": -5.331966400146484,
      "step": 1434
    },
    {
      "epoch": 1.88,
      "learning_rate": 4.6231200411761444e-05,
      "logits/chosen": -2.464130401611328,
      "logits/rejected": -2.5502843856811523,
      "logps/chosen": -243.13787841796875,
      "logps/rejected": -283.830322265625,
      "loss": 0.04,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.265555500984192,
      "rewards/margins": 5.3822526931762695,
      "rewards/rejected": -6.647809028625488,
      "step": 1435
    },
    {
      "epoch": 1.88,
      "learning_rate": 4.622569875968629e-05,
      "logits/chosen": -2.321964979171753,
      "logits/rejected": -2.423978805541992,
      "logps/chosen": -205.4765625,
      "logps/rejected": -242.30911254882812,
      "loss": 0.1069,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7557665705680847,
      "rewards/margins": 4.173945903778076,
      "rewards/rejected": -4.929712772369385,
      "step": 1436
    },
    {
      "epoch": 1.88,
      "learning_rate": 4.6220193422767665e-05,
      "logits/chosen": -2.3185129165649414,
      "logits/rejected": -2.246967077255249,
      "logps/chosen": -190.0450439453125,
      "logps/rejected": -195.37098693847656,
      "loss": 0.1683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.310142159461975,
      "rewards/margins": 3.645517587661743,
      "rewards/rejected": -4.955659866333008,
      "step": 1437
    },
    {
      "epoch": 1.88,
      "learning_rate": 4.6214684401961314e-05,
      "logits/chosen": -2.327990770339966,
      "logits/rejected": -2.4045562744140625,
      "logps/chosen": -195.2279052734375,
      "logps/rejected": -307.27642822265625,
      "loss": 0.1309,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.452347993850708,
      "rewards/margins": 4.286505222320557,
      "rewards/rejected": -5.738852500915527,
      "step": 1438
    },
    {
      "epoch": 1.88,
      "learning_rate": 4.620917169822363e-05,
      "logits/chosen": -2.33237361907959,
      "logits/rejected": -2.329893112182617,
      "logps/chosen": -197.7900390625,
      "logps/rejected": -215.33221435546875,
      "loss": 0.3258,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.448103666305542,
      "rewards/margins": 3.1846346855163574,
      "rewards/rejected": -4.63273811340332,
      "step": 1439
    },
    {
      "epoch": 1.88,
      "learning_rate": 4.6203655312511616e-05,
      "logits/chosen": -2.4939942359924316,
      "logits/rejected": -2.5082345008850098,
      "logps/chosen": -257.9974060058594,
      "logps/rejected": -305.877197265625,
      "loss": 0.2161,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5948214530944824,
      "rewards/margins": 3.7099242210388184,
      "rewards/rejected": -5.304745197296143,
      "step": 1440
    },
    {
      "epoch": 1.89,
      "learning_rate": 4.619813524578295e-05,
      "logits/chosen": -2.354602813720703,
      "logits/rejected": -2.3939921855926514,
      "logps/chosen": -215.87257385253906,
      "logps/rejected": -281.49603271484375,
      "loss": 0.0665,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.500991940498352,
      "rewards/margins": 4.147805690765381,
      "rewards/rejected": -5.648797512054443,
      "step": 1441
    },
    {
      "epoch": 1.89,
      "learning_rate": 4.619261149899592e-05,
      "logits/chosen": -2.2306058406829834,
      "logits/rejected": -2.329472780227661,
      "logps/chosen": -195.59654235839844,
      "logps/rejected": -269.08892822265625,
      "loss": 0.1455,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8193372488021851,
      "rewards/margins": 4.419328212738037,
      "rewards/rejected": -5.23866605758667,
      "step": 1442
    },
    {
      "epoch": 1.89,
      "learning_rate": 4.618708407310947e-05,
      "logits/chosen": -2.4757118225097656,
      "logits/rejected": -2.430291175842285,
      "logps/chosen": -287.3864440917969,
      "logps/rejected": -302.011962890625,
      "loss": 0.1739,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6464817523956299,
      "rewards/margins": 3.5614006519317627,
      "rewards/rejected": -5.207882881164551,
      "step": 1443
    },
    {
      "epoch": 1.89,
      "learning_rate": 4.6181552969083165e-05,
      "logits/chosen": -2.157444715499878,
      "logits/rejected": -2.1985301971435547,
      "logps/chosen": -194.09764099121094,
      "logps/rejected": -267.64715576171875,
      "loss": 0.1754,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1710000038146973,
      "rewards/margins": 3.7776384353637695,
      "rewards/rejected": -4.948638439178467,
      "step": 1444
    },
    {
      "epoch": 1.89,
      "learning_rate": 4.617601818787724e-05,
      "logits/chosen": -1.9728646278381348,
      "logits/rejected": -2.009904623031616,
      "logps/chosen": -160.30093383789062,
      "logps/rejected": -225.08453369140625,
      "loss": 0.1576,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.134428858757019,
      "rewards/margins": 4.066022872924805,
      "rewards/rejected": -5.200451374053955,
      "step": 1445
    },
    {
      "epoch": 1.89,
      "learning_rate": 4.617047973045254e-05,
      "logits/chosen": -2.374601125717163,
      "logits/rejected": -2.431764841079712,
      "logps/chosen": -255.8111114501953,
      "logps/rejected": -308.56298828125,
      "loss": 0.0817,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9789934158325195,
      "rewards/margins": 5.199550628662109,
      "rewards/rejected": -6.178544044494629,
      "step": 1446
    },
    {
      "epoch": 1.89,
      "learning_rate": 4.6164937597770555e-05,
      "logits/chosen": -2.4467074871063232,
      "logits/rejected": -2.5327441692352295,
      "logps/chosen": -227.62490844726562,
      "logps/rejected": -285.66241455078125,
      "loss": 0.0685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1844345331192017,
      "rewards/margins": 4.270139217376709,
      "rewards/rejected": -5.454573631286621,
      "step": 1447
    },
    {
      "epoch": 1.89,
      "learning_rate": 4.615939179079342e-05,
      "logits/chosen": -1.8311939239501953,
      "logits/rejected": -1.9591543674468994,
      "logps/chosen": -181.0715789794922,
      "logps/rejected": -255.28375244140625,
      "loss": 0.2963,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9211238622665405,
      "rewards/margins": 4.1576642990112305,
      "rewards/rejected": -6.078787803649902,
      "step": 1448
    },
    {
      "epoch": 1.9,
      "learning_rate": 4.615384231048391e-05,
      "logits/chosen": -2.24050235748291,
      "logits/rejected": -2.2615864276885986,
      "logps/chosen": -227.12237548828125,
      "logps/rejected": -235.6379852294922,
      "loss": 0.2308,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8242486715316772,
      "rewards/margins": 3.2859408855438232,
      "rewards/rejected": -5.110189437866211,
      "step": 1449
    },
    {
      "epoch": 1.9,
      "learning_rate": 4.6148289157805406e-05,
      "logits/chosen": -2.395170211791992,
      "logits/rejected": -2.547234535217285,
      "logps/chosen": -170.02056884765625,
      "logps/rejected": -283.0645751953125,
      "loss": 0.0816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.996381402015686,
      "rewards/margins": 3.851898670196533,
      "rewards/rejected": -5.848280429840088,
      "step": 1450
    },
    {
      "epoch": 1.9,
      "learning_rate": 4.614273233372198e-05,
      "logits/chosen": -2.4565505981445312,
      "logits/rejected": -2.4085004329681396,
      "logps/chosen": -275.6915283203125,
      "logps/rejected": -321.6641845703125,
      "loss": 0.219,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4337880611419678,
      "rewards/margins": 4.953248023986816,
      "rewards/rejected": -6.387036323547363,
      "step": 1451
    },
    {
      "epoch": 1.9,
      "learning_rate": 4.61371718391983e-05,
      "logits/chosen": -2.3917980194091797,
      "logits/rejected": -2.414000988006592,
      "logps/chosen": -201.67922973632812,
      "logps/rejected": -291.4822998046875,
      "loss": 0.0235,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.536219596862793,
      "rewards/margins": 5.365948677062988,
      "rewards/rejected": -6.902168273925781,
      "step": 1452
    },
    {
      "epoch": 1.9,
      "learning_rate": 4.6131607675199686e-05,
      "logits/chosen": -2.4047164916992188,
      "logits/rejected": -2.3522493839263916,
      "logps/chosen": -228.041748046875,
      "logps/rejected": -243.53179931640625,
      "loss": 0.1546,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.3639187812805176,
      "rewards/margins": 3.640273332595825,
      "rewards/rejected": -6.004191875457764,
      "step": 1453
    },
    {
      "epoch": 1.9,
      "learning_rate": 4.612603984269208e-05,
      "logits/chosen": -2.2534050941467285,
      "logits/rejected": -2.179338216781616,
      "logps/chosen": -238.51016235351562,
      "logps/rejected": -278.68157958984375,
      "loss": 0.2575,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.542654037475586,
      "rewards/margins": 4.30103874206543,
      "rewards/rejected": -6.843692779541016,
      "step": 1454
    },
    {
      "epoch": 1.9,
      "learning_rate": 4.61204683426421e-05,
      "logits/chosen": -2.132852077484131,
      "logits/rejected": -2.1885266304016113,
      "logps/chosen": -198.23362731933594,
      "logps/rejected": -253.08970642089844,
      "loss": 0.2702,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.3060522079467773,
      "rewards/margins": 3.5543110370635986,
      "rewards/rejected": -5.860363006591797,
      "step": 1455
    },
    {
      "epoch": 1.91,
      "learning_rate": 4.611489317601696e-05,
      "logits/chosen": -2.1477696895599365,
      "logits/rejected": -2.235487937927246,
      "logps/chosen": -261.10101318359375,
      "logps/rejected": -321.97259521484375,
      "loss": 0.0589,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.0646214485168457,
      "rewards/margins": 4.98212194442749,
      "rewards/rejected": -7.046743392944336,
      "step": 1456
    },
    {
      "epoch": 1.91,
      "learning_rate": 4.6109314343784524e-05,
      "logits/chosen": -2.1931657791137695,
      "logits/rejected": -2.2348809242248535,
      "logps/chosen": -168.6276397705078,
      "logps/rejected": -263.44561767578125,
      "loss": 0.1359,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.65928053855896,
      "rewards/margins": 3.9209232330322266,
      "rewards/rejected": -5.580203533172607,
      "step": 1457
    },
    {
      "epoch": 1.91,
      "learning_rate": 4.61037318469133e-05,
      "logits/chosen": -2.286311149597168,
      "logits/rejected": -2.3383705615997314,
      "logps/chosen": -178.47718811035156,
      "logps/rejected": -235.9649200439453,
      "loss": 0.2152,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8744279146194458,
      "rewards/margins": 3.764946937561035,
      "rewards/rejected": -5.63937520980835,
      "step": 1458
    },
    {
      "epoch": 1.91,
      "learning_rate": 4.6098145686372415e-05,
      "logits/chosen": -1.8387361764907837,
      "logits/rejected": -1.8523186445236206,
      "logps/chosen": -215.5361328125,
      "logps/rejected": -265.8430480957031,
      "loss": 0.1596,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.201512575149536,
      "rewards/margins": 3.9642109870910645,
      "rewards/rejected": -6.16572380065918,
      "step": 1459
    },
    {
      "epoch": 1.91,
      "learning_rate": 4.609255586313166e-05,
      "logits/chosen": -2.1031899452209473,
      "logits/rejected": -2.1518192291259766,
      "logps/chosen": -231.7281494140625,
      "logps/rejected": -219.5949249267578,
      "loss": 0.1425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.150951385498047,
      "rewards/margins": 3.822639226913452,
      "rewards/rejected": -5.973590850830078,
      "step": 1460
    },
    {
      "epoch": 1.91,
      "learning_rate": 4.608696237816143e-05,
      "logits/chosen": -2.3749542236328125,
      "logits/rejected": -2.404801845550537,
      "logps/chosen": -190.76507568359375,
      "logps/rejected": -212.51536560058594,
      "loss": 0.2917,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.243725061416626,
      "rewards/margins": 3.3376617431640625,
      "rewards/rejected": -5.581386566162109,
      "step": 1461
    },
    {
      "epoch": 1.91,
      "learning_rate": 4.6081365232432766e-05,
      "logits/chosen": -2.2417476177215576,
      "logits/rejected": -2.29201602935791,
      "logps/chosen": -309.34808349609375,
      "logps/rejected": -365.32110595703125,
      "loss": 0.1465,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.222254753112793,
      "rewards/margins": 4.027994155883789,
      "rewards/rejected": -7.250248908996582,
      "step": 1462
    },
    {
      "epoch": 1.91,
      "learning_rate": 4.607576442691737e-05,
      "logits/chosen": -2.15925931930542,
      "logits/rejected": -2.148500919342041,
      "logps/chosen": -222.72410583496094,
      "logps/rejected": -263.0013427734375,
      "loss": 0.2245,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.0160765647888184,
      "rewards/margins": 4.10772705078125,
      "rewards/rejected": -6.12380313873291,
      "step": 1463
    },
    {
      "epoch": 1.92,
      "learning_rate": 4.607015996258753e-05,
      "logits/chosen": -2.2752912044525146,
      "logits/rejected": -2.4492502212524414,
      "logps/chosen": -156.12005615234375,
      "logps/rejected": -226.24057006835938,
      "loss": 0.15,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.519330382347107,
      "rewards/margins": 4.664186477661133,
      "rewards/rejected": -6.183516979217529,
      "step": 1464
    },
    {
      "epoch": 1.92,
      "learning_rate": 4.606455184041622e-05,
      "logits/chosen": -2.2248048782348633,
      "logits/rejected": -2.3291237354278564,
      "logps/chosen": -167.07228088378906,
      "logps/rejected": -236.86578369140625,
      "loss": 0.2488,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.011113166809082,
      "rewards/margins": 3.412134885787964,
      "rewards/rejected": -5.423248767852783,
      "step": 1465
    },
    {
      "epoch": 1.92,
      "learning_rate": 4.6058940061377034e-05,
      "logits/chosen": -2.4281818866729736,
      "logits/rejected": -2.378067970275879,
      "logps/chosen": -217.27587890625,
      "logps/rejected": -243.82916259765625,
      "loss": 0.3441,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.679200291633606,
      "rewards/margins": 3.3170528411865234,
      "rewards/rejected": -4.996253490447998,
      "step": 1466
    },
    {
      "epoch": 1.92,
      "learning_rate": 4.605332462644417e-05,
      "logits/chosen": -2.0087335109710693,
      "logits/rejected": -2.0646257400512695,
      "logps/chosen": -196.29052734375,
      "logps/rejected": -237.742431640625,
      "loss": 0.3309,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7196638584136963,
      "rewards/margins": 3.8741507530212402,
      "rewards/rejected": -5.593814373016357,
      "step": 1467
    },
    {
      "epoch": 1.92,
      "learning_rate": 4.604770553659249e-05,
      "logits/chosen": -2.3972527980804443,
      "logits/rejected": -2.4377150535583496,
      "logps/chosen": -185.716796875,
      "logps/rejected": -259.7029113769531,
      "loss": 0.1088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.643090844154358,
      "rewards/margins": 3.906405448913574,
      "rewards/rejected": -5.549496173858643,
      "step": 1468
    },
    {
      "epoch": 1.92,
      "learning_rate": 4.604208279279749e-05,
      "logits/chosen": -2.144949436187744,
      "logits/rejected": -2.169041156768799,
      "logps/chosen": -160.50648498535156,
      "logps/rejected": -240.07061767578125,
      "loss": 0.2066,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6182748079299927,
      "rewards/margins": 3.4854378700256348,
      "rewards/rejected": -5.103713035583496,
      "step": 1469
    },
    {
      "epoch": 1.92,
      "learning_rate": 4.6036456396035294e-05,
      "logits/chosen": -2.12722110748291,
      "logits/rejected": -2.192887783050537,
      "logps/chosen": -230.61842346191406,
      "logps/rejected": -281.3009338378906,
      "loss": 0.1416,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.241001605987549,
      "rewards/margins": 4.063772678375244,
      "rewards/rejected": -6.304775238037109,
      "step": 1470
    },
    {
      "epoch": 1.93,
      "learning_rate": 4.603082634728266e-05,
      "logits/chosen": -2.2297396659851074,
      "logits/rejected": -2.20890736579895,
      "logps/chosen": -216.92306518554688,
      "logps/rejected": -302.16015625,
      "loss": 0.0356,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4535167217254639,
      "rewards/margins": 5.6094746589660645,
      "rewards/rejected": -7.062991619110107,
      "step": 1471
    },
    {
      "epoch": 1.93,
      "learning_rate": 4.602519264751697e-05,
      "logits/chosen": -2.5697968006134033,
      "logits/rejected": -2.4660468101501465,
      "logps/chosen": -272.8087158203125,
      "logps/rejected": -276.1620788574219,
      "loss": 0.2615,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8285648822784424,
      "rewards/margins": 3.2634503841400146,
      "rewards/rejected": -5.092015266418457,
      "step": 1472
    },
    {
      "epoch": 1.93,
      "learning_rate": 4.601955529771628e-05,
      "logits/chosen": -2.385674476623535,
      "logits/rejected": -2.5492734909057617,
      "logps/chosen": -209.6891632080078,
      "logps/rejected": -247.408935546875,
      "loss": 0.175,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8343769311904907,
      "rewards/margins": 3.4775278568267822,
      "rewards/rejected": -5.311904430389404,
      "step": 1473
    },
    {
      "epoch": 1.93,
      "learning_rate": 4.601391429885922e-05,
      "logits/chosen": -2.5167958736419678,
      "logits/rejected": -2.5103375911712646,
      "logps/chosen": -239.463623046875,
      "logps/rejected": -271.7610778808594,
      "loss": 0.1614,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7672418355941772,
      "rewards/margins": 4.362864971160889,
      "rewards/rejected": -6.130106449127197,
      "step": 1474
    },
    {
      "epoch": 1.93,
      "learning_rate": 4.600826965192509e-05,
      "logits/chosen": -2.1353819370269775,
      "logits/rejected": -2.2745490074157715,
      "logps/chosen": -144.68832397460938,
      "logps/rejected": -287.36920166015625,
      "loss": 0.0856,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7524394989013672,
      "rewards/margins": 4.990352630615234,
      "rewards/rejected": -6.742793083190918,
      "step": 1475
    },
    {
      "epoch": 1.93,
      "learning_rate": 4.6002621357893826e-05,
      "logits/chosen": -2.486030340194702,
      "logits/rejected": -2.4928431510925293,
      "logps/chosen": -223.55197143554688,
      "logps/rejected": -286.7210693359375,
      "loss": 0.0372,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6188063621520996,
      "rewards/margins": 5.330704212188721,
      "rewards/rejected": -6.9495110511779785,
      "step": 1476
    },
    {
      "epoch": 1.93,
      "learning_rate": 4.5996969417745986e-05,
      "logits/chosen": -2.2136309146881104,
      "logits/rejected": -2.255204200744629,
      "logps/chosen": -209.3388214111328,
      "logps/rejected": -260.0314025878906,
      "loss": 0.1217,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.289798378944397,
      "rewards/margins": 4.679617404937744,
      "rewards/rejected": -5.96941614151001,
      "step": 1477
    },
    {
      "epoch": 1.93,
      "learning_rate": 4.599131383246277e-05,
      "logits/chosen": -2.519814968109131,
      "logits/rejected": -2.547168254852295,
      "logps/chosen": -209.82684326171875,
      "logps/rejected": -275.3058166503906,
      "loss": 0.1756,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6560988426208496,
      "rewards/margins": 4.22465705871582,
      "rewards/rejected": -5.880756378173828,
      "step": 1478
    },
    {
      "epoch": 1.94,
      "learning_rate": 4.598565460302599e-05,
      "logits/chosen": -2.0395500659942627,
      "logits/rejected": -2.038343906402588,
      "logps/chosen": -179.4903564453125,
      "logps/rejected": -207.68734741210938,
      "loss": 0.2468,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0823073387145996,
      "rewards/margins": 4.204888820648193,
      "rewards/rejected": -5.287196159362793,
      "step": 1479
    },
    {
      "epoch": 1.94,
      "learning_rate": 4.5979991730418105e-05,
      "logits/chosen": -2.3252034187316895,
      "logits/rejected": -2.3550727367401123,
      "logps/chosen": -188.61257934570312,
      "logps/rejected": -251.78277587890625,
      "loss": 0.1684,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.603855848312378,
      "rewards/margins": 4.1132965087890625,
      "rewards/rejected": -5.7171525955200195,
      "step": 1480
    },
    {
      "epoch": 1.94,
      "learning_rate": 4.5974325215622225e-05,
      "logits/chosen": -2.103459119796753,
      "logits/rejected": -2.1562860012054443,
      "logps/chosen": -180.7230987548828,
      "logps/rejected": -237.24388122558594,
      "loss": 0.2603,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9204591512680054,
      "rewards/margins": 2.8535118103027344,
      "rewards/rejected": -4.773970603942871,
      "step": 1481
    },
    {
      "epoch": 1.94,
      "learning_rate": 4.596865505962205e-05,
      "logits/chosen": -2.130312919616699,
      "logits/rejected": -2.2995500564575195,
      "logps/chosen": -201.30421447753906,
      "logps/rejected": -257.9309387207031,
      "loss": 0.212,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.0479345321655273,
      "rewards/margins": 3.392012596130371,
      "rewards/rejected": -5.439947128295898,
      "step": 1482
    },
    {
      "epoch": 1.94,
      "learning_rate": 4.596298126340195e-05,
      "logits/chosen": -2.275320291519165,
      "logits/rejected": -2.270385980606079,
      "logps/chosen": -256.5345458984375,
      "logps/rejected": -315.8306884765625,
      "loss": 0.0989,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.094556450843811,
      "rewards/margins": 4.61052131652832,
      "rewards/rejected": -5.705077648162842,
      "step": 1483
    },
    {
      "epoch": 1.94,
      "learning_rate": 4.595730382794691e-05,
      "logits/chosen": -1.9736368656158447,
      "logits/rejected": -2.069291353225708,
      "logps/chosen": -214.57188415527344,
      "logps/rejected": -224.57400512695312,
      "loss": 0.2066,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7135539054870605,
      "rewards/margins": 3.323864459991455,
      "rewards/rejected": -5.037418842315674,
      "step": 1484
    },
    {
      "epoch": 1.94,
      "learning_rate": 4.595162275424255e-05,
      "logits/chosen": -2.2768514156341553,
      "logits/rejected": -2.439985513687134,
      "logps/chosen": -168.1714630126953,
      "logps/rejected": -264.5210876464844,
      "loss": 0.046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3404206037521362,
      "rewards/margins": 4.816141128540039,
      "rewards/rejected": -6.156561374664307,
      "step": 1485
    },
    {
      "epoch": 1.94,
      "learning_rate": 4.594593804327513e-05,
      "logits/chosen": -2.3617050647735596,
      "logits/rejected": -2.4006364345550537,
      "logps/chosen": -223.51766967773438,
      "logps/rejected": -264.58575439453125,
      "loss": 0.1137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4779276847839355,
      "rewards/margins": 3.821845054626465,
      "rewards/rejected": -5.299773216247559,
      "step": 1486
    },
    {
      "epoch": 1.95,
      "learning_rate": 4.594024969603151e-05,
      "logits/chosen": -2.4014294147491455,
      "logits/rejected": -2.345073938369751,
      "logps/chosen": -219.4610137939453,
      "logps/rejected": -262.7455749511719,
      "loss": 0.0748,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.476982831954956,
      "rewards/margins": 3.7781622409820557,
      "rewards/rejected": -5.2551445960998535,
      "step": 1487
    },
    {
      "epoch": 1.95,
      "learning_rate": 4.5934557713499214e-05,
      "logits/chosen": -2.1576740741729736,
      "logits/rejected": -2.2681870460510254,
      "logps/chosen": -248.980712890625,
      "logps/rejected": -353.2499694824219,
      "loss": 0.1602,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.409653902053833,
      "rewards/margins": 5.4672088623046875,
      "rewards/rejected": -6.8768630027771,
      "step": 1488
    },
    {
      "epoch": 1.95,
      "learning_rate": 4.592886209666639e-05,
      "logits/chosen": -2.3278369903564453,
      "logits/rejected": -2.4039418697357178,
      "logps/chosen": -175.85903930664062,
      "logps/rejected": -243.25254821777344,
      "loss": 0.0638,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.090004563331604,
      "rewards/margins": 4.7786030769348145,
      "rewards/rejected": -5.868607997894287,
      "step": 1489
    },
    {
      "epoch": 1.95,
      "learning_rate": 4.5923162846521824e-05,
      "logits/chosen": -2.2458479404449463,
      "logits/rejected": -2.3095836639404297,
      "logps/chosen": -252.07955932617188,
      "logps/rejected": -280.0485534667969,
      "loss": 0.2264,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.0053060054779053,
      "rewards/margins": 3.449532985687256,
      "rewards/rejected": -5.454838752746582,
      "step": 1490
    },
    {
      "epoch": 1.95,
      "learning_rate": 4.5917459964054896e-05,
      "logits/chosen": -2.519906997680664,
      "logits/rejected": -2.511167049407959,
      "logps/chosen": -198.07257080078125,
      "logps/rejected": -263.76898193359375,
      "loss": 0.1654,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.930348813533783,
      "rewards/margins": 4.16477632522583,
      "rewards/rejected": -5.095125675201416,
      "step": 1491
    },
    {
      "epoch": 1.95,
      "learning_rate": 4.5911753450255665e-05,
      "logits/chosen": -2.2251136302948,
      "logits/rejected": -2.2730202674865723,
      "logps/chosen": -236.90675354003906,
      "logps/rejected": -275.7073059082031,
      "loss": 0.1027,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.889454960823059,
      "rewards/margins": 3.976149082183838,
      "rewards/rejected": -5.865603446960449,
      "step": 1492
    },
    {
      "epoch": 1.95,
      "learning_rate": 4.59060433061148e-05,
      "logits/chosen": -2.395418643951416,
      "logits/rejected": -2.4963815212249756,
      "logps/chosen": -207.62921142578125,
      "logps/rejected": -221.30831909179688,
      "loss": 0.2778,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6155648231506348,
      "rewards/margins": 3.7796285152435303,
      "rewards/rejected": -5.395193099975586,
      "step": 1493
    },
    {
      "epoch": 1.96,
      "learning_rate": 4.5900329532623585e-05,
      "logits/chosen": -2.3171846866607666,
      "logits/rejected": -2.2144782543182373,
      "logps/chosen": -192.01412963867188,
      "logps/rejected": -202.37173461914062,
      "loss": 0.4487,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.814784288406372,
      "rewards/margins": 2.9413533210754395,
      "rewards/rejected": -4.756137847900391,
      "step": 1494
    },
    {
      "epoch": 1.96,
      "learning_rate": 4.589461213077395e-05,
      "logits/chosen": -1.8302745819091797,
      "logits/rejected": -1.8340033292770386,
      "logps/chosen": -185.82254028320312,
      "logps/rejected": -226.58468627929688,
      "loss": 0.2119,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3006807565689087,
      "rewards/margins": 3.979428291320801,
      "rewards/rejected": -5.280109882354736,
      "step": 1495
    },
    {
      "epoch": 1.96,
      "learning_rate": 4.588889110155845e-05,
      "logits/chosen": -2.277804374694824,
      "logits/rejected": -2.3033041954040527,
      "logps/chosen": -245.10751342773438,
      "logps/rejected": -303.5085754394531,
      "loss": 0.1516,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6710920333862305,
      "rewards/margins": 4.150707244873047,
      "rewards/rejected": -5.821799278259277,
      "step": 1496
    },
    {
      "epoch": 1.96,
      "learning_rate": 4.5883166445970296e-05,
      "logits/chosen": -2.103105068206787,
      "logits/rejected": -2.114781618118286,
      "logps/chosen": -219.78988647460938,
      "logps/rejected": -306.8294982910156,
      "loss": 0.2936,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6369211673736572,
      "rewards/margins": 2.928783893585205,
      "rewards/rejected": -4.565704822540283,
      "step": 1497
    },
    {
      "epoch": 1.96,
      "learning_rate": 4.587743816500328e-05,
      "logits/chosen": -2.03912091255188,
      "logits/rejected": -2.0565059185028076,
      "logps/chosen": -232.88314819335938,
      "logps/rejected": -227.30206298828125,
      "loss": 0.1022,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0165839195251465,
      "rewards/margins": 3.8692049980163574,
      "rewards/rejected": -4.885788917541504,
      "step": 1498
    },
    {
      "epoch": 1.96,
      "learning_rate": 4.587170625965185e-05,
      "logits/chosen": -1.9884145259857178,
      "logits/rejected": -1.9488219022750854,
      "logps/chosen": -204.64010620117188,
      "logps/rejected": -283.6075134277344,
      "loss": 0.15,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.870671033859253,
      "rewards/margins": 4.004744529724121,
      "rewards/rejected": -5.875415802001953,
      "step": 1499
    },
    {
      "epoch": 1.96,
      "learning_rate": 4.586597073091109e-05,
      "logits/chosen": -2.0228824615478516,
      "logits/rejected": -2.1101438999176025,
      "logps/chosen": -197.7698211669922,
      "logps/rejected": -271.5038757324219,
      "loss": 0.1237,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7399284243583679,
      "rewards/margins": 3.924260139465332,
      "rewards/rejected": -4.664188861846924,
      "step": 1500
    },
    {
      "epoch": 1.96,
      "learning_rate": 4.586023157977671e-05,
      "logits/chosen": -2.292987585067749,
      "logits/rejected": -2.3368139266967773,
      "logps/chosen": -216.46487426757812,
      "logps/rejected": -270.67730712890625,
      "loss": 0.1502,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.036461591720581,
      "rewards/margins": 3.9408953189849854,
      "rewards/rejected": -5.977356910705566,
      "step": 1501
    },
    {
      "epoch": 1.97,
      "learning_rate": 4.5854488807245025e-05,
      "logits/chosen": -2.458857774734497,
      "logits/rejected": -2.3832597732543945,
      "logps/chosen": -200.65383911132812,
      "logps/rejected": -245.40292358398438,
      "loss": 0.1366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4562873840332031,
      "rewards/margins": 3.750645637512207,
      "rewards/rejected": -5.20693302154541,
      "step": 1502
    },
    {
      "epoch": 1.97,
      "learning_rate": 4.584874241431301e-05,
      "logits/chosen": -2.4760890007019043,
      "logits/rejected": -2.3838562965393066,
      "logps/chosen": -230.46986389160156,
      "logps/rejected": -249.74501037597656,
      "loss": 0.2681,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.6505513191223145,
      "rewards/margins": 3.109177589416504,
      "rewards/rejected": -4.75972843170166,
      "step": 1503
    },
    {
      "epoch": 1.97,
      "learning_rate": 4.5842992401978255e-05,
      "logits/chosen": -2.338324785232544,
      "logits/rejected": -2.331594705581665,
      "logps/chosen": -207.29168701171875,
      "logps/rejected": -252.10321044921875,
      "loss": 0.1927,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0485973358154297,
      "rewards/margins": 3.800584316253662,
      "rewards/rejected": -4.84918212890625,
      "step": 1504
    },
    {
      "epoch": 1.97,
      "learning_rate": 4.5837238771238975e-05,
      "logits/chosen": -2.3720309734344482,
      "logits/rejected": -2.443889856338501,
      "logps/chosen": -230.22865295410156,
      "logps/rejected": -280.8548889160156,
      "loss": 0.0657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3540129661560059,
      "rewards/margins": 4.325744152069092,
      "rewards/rejected": -5.6797566413879395,
      "step": 1505
    },
    {
      "epoch": 1.97,
      "learning_rate": 4.583148152309403e-05,
      "logits/chosen": -2.3643789291381836,
      "logits/rejected": -2.5145559310913086,
      "logps/chosen": -185.69049072265625,
      "logps/rejected": -248.0831756591797,
      "loss": 0.207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0387358665466309,
      "rewards/margins": 3.8081166744232178,
      "rewards/rejected": -4.846852779388428,
      "step": 1506
    },
    {
      "epoch": 1.97,
      "learning_rate": 4.582572065854287e-05,
      "logits/chosen": -2.0276918411254883,
      "logits/rejected": -1.997923493385315,
      "logps/chosen": -213.56065368652344,
      "logps/rejected": -217.77130126953125,
      "loss": 0.3128,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1286025047302246,
      "rewards/margins": 1.9039826393127441,
      "rewards/rejected": -4.032585144042969,
      "step": 1507
    },
    {
      "epoch": 1.97,
      "learning_rate": 4.581995617858562e-05,
      "logits/chosen": -2.252142906188965,
      "logits/rejected": -2.3322079181671143,
      "logps/chosen": -168.25584411621094,
      "logps/rejected": -226.796630859375,
      "loss": 0.1596,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.414211630821228,
      "rewards/margins": 3.2070906162261963,
      "rewards/rejected": -4.621302604675293,
      "step": 1508
    },
    {
      "epoch": 1.97,
      "learning_rate": 4.5814188084223006e-05,
      "logits/chosen": -2.3944835662841797,
      "logits/rejected": -2.393568515777588,
      "logps/chosen": -191.02215576171875,
      "logps/rejected": -210.15518188476562,
      "loss": 0.3093,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.204001545906067,
      "rewards/margins": 3.173072338104248,
      "rewards/rejected": -4.377073764801025,
      "step": 1509
    },
    {
      "epoch": 1.98,
      "learning_rate": 4.580841637645638e-05,
      "logits/chosen": -2.025181531906128,
      "logits/rejected": -2.0204172134399414,
      "logps/chosen": -163.89816284179688,
      "logps/rejected": -185.0117645263672,
      "loss": 0.2107,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1706594228744507,
      "rewards/margins": 2.758812427520752,
      "rewards/rejected": -3.929471731185913,
      "step": 1510
    },
    {
      "epoch": 1.98,
      "learning_rate": 4.5802641056287724e-05,
      "logits/chosen": -2.3880228996276855,
      "logits/rejected": -2.4138453006744385,
      "logps/chosen": -217.76901245117188,
      "logps/rejected": -264.3223876953125,
      "loss": 0.2101,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1917675733566284,
      "rewards/margins": 3.443121910095215,
      "rewards/rejected": -4.634889602661133,
      "step": 1511
    },
    {
      "epoch": 1.98,
      "learning_rate": 4.5796862124719663e-05,
      "logits/chosen": -2.2867062091827393,
      "logits/rejected": -2.23726487159729,
      "logps/chosen": -228.87283325195312,
      "logps/rejected": -269.2378845214844,
      "loss": 0.0988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0112980604171753,
      "rewards/margins": 4.46527099609375,
      "rewards/rejected": -5.476568698883057,
      "step": 1512
    },
    {
      "epoch": 1.98,
      "learning_rate": 4.5791079582755434e-05,
      "logits/chosen": -2.4388363361358643,
      "logits/rejected": -2.466726779937744,
      "logps/chosen": -193.21365356445312,
      "logps/rejected": -249.07179260253906,
      "loss": 0.1101,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1325361728668213,
      "rewards/margins": 3.8823916912078857,
      "rewards/rejected": -5.014928340911865,
      "step": 1513
    },
    {
      "epoch": 1.98,
      "learning_rate": 4.578529343139889e-05,
      "logits/chosen": -2.3350844383239746,
      "logits/rejected": -2.263835906982422,
      "logps/chosen": -151.61380004882812,
      "logps/rejected": -199.18035888671875,
      "loss": 0.2984,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3873097896575928,
      "rewards/margins": 2.5545272827148438,
      "rewards/rejected": -3.9418370723724365,
      "step": 1514
    },
    {
      "epoch": 1.98,
      "learning_rate": 4.5779503671654544e-05,
      "logits/chosen": -2.3566839694976807,
      "logits/rejected": -2.525066375732422,
      "logps/chosen": -205.33551025390625,
      "logps/rejected": -275.2447204589844,
      "loss": 0.0893,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1993653774261475,
      "rewards/margins": 4.662095069885254,
      "rewards/rejected": -5.861460208892822,
      "step": 1515
    },
    {
      "epoch": 1.98,
      "learning_rate": 4.5773710304527493e-05,
      "logits/chosen": -2.0783872604370117,
      "logits/rejected": -2.117863893508911,
      "logps/chosen": -190.528076171875,
      "logps/rejected": -224.16598510742188,
      "loss": 0.2068,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4108234643936157,
      "rewards/margins": 2.3509793281555176,
      "rewards/rejected": -3.761802911758423,
      "step": 1516
    },
    {
      "epoch": 1.99,
      "learning_rate": 4.5767913331023506e-05,
      "logits/chosen": -2.3093221187591553,
      "logits/rejected": -2.437835693359375,
      "logps/chosen": -177.2186737060547,
      "logps/rejected": -241.48597717285156,
      "loss": 0.1371,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2239621877670288,
      "rewards/margins": 3.5838568210601807,
      "rewards/rejected": -4.80781888961792,
      "step": 1517
    },
    {
      "epoch": 1.99,
      "learning_rate": 4.576211275214894e-05,
      "logits/chosen": -2.2390902042388916,
      "logits/rejected": -2.2308666706085205,
      "logps/chosen": -193.17189025878906,
      "logps/rejected": -195.62255859375,
      "loss": 0.1861,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9776548147201538,
      "rewards/margins": 3.1817262172698975,
      "rewards/rejected": -4.15938138961792,
      "step": 1518
    },
    {
      "epoch": 1.99,
      "learning_rate": 4.5756308568910786e-05,
      "logits/chosen": -2.1870338916778564,
      "logits/rejected": -2.4006149768829346,
      "logps/chosen": -186.26927185058594,
      "logps/rejected": -255.21742248535156,
      "loss": 0.2058,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5204411745071411,
      "rewards/margins": 3.8511171340942383,
      "rewards/rejected": -5.371558666229248,
      "step": 1519
    },
    {
      "epoch": 1.99,
      "learning_rate": 4.5750500782316684e-05,
      "logits/chosen": -2.27748966217041,
      "logits/rejected": -2.291846752166748,
      "logps/chosen": -179.44192504882812,
      "logps/rejected": -219.5472412109375,
      "loss": 0.2301,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0973180532455444,
      "rewards/margins": 2.7823243141174316,
      "rewards/rejected": -3.8796424865722656,
      "step": 1520
    },
    {
      "epoch": 1.99,
      "learning_rate": 4.574468939337488e-05,
      "logits/chosen": -2.2131187915802,
      "logits/rejected": -2.104618549346924,
      "logps/chosen": -228.83322143554688,
      "logps/rejected": -260.3489074707031,
      "loss": 0.167,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7398812770843506,
      "rewards/margins": 3.2543160915374756,
      "rewards/rejected": -3.994197130203247,
      "step": 1521
    },
    {
      "epoch": 1.99,
      "learning_rate": 4.573887440309424e-05,
      "logits/chosen": -2.2128376960754395,
      "logits/rejected": -2.3602702617645264,
      "logps/chosen": -169.43557739257812,
      "logps/rejected": -223.89573669433594,
      "loss": 0.2419,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.455782413482666,
      "rewards/margins": 3.0371322631835938,
      "rewards/rejected": -4.49291467666626,
      "step": 1522
    },
    {
      "epoch": 1.99,
      "learning_rate": 4.5733055812484263e-05,
      "logits/chosen": -2.367260456085205,
      "logits/rejected": -2.4905948638916016,
      "logps/chosen": -256.87579345703125,
      "logps/rejected": -341.8155212402344,
      "loss": 0.0541,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6214128732681274,
      "rewards/margins": 4.6316304206848145,
      "rewards/rejected": -6.253043174743652,
      "step": 1523
    },
    {
      "epoch": 1.99,
      "learning_rate": 4.5727233622555076e-05,
      "logits/chosen": -2.360616445541382,
      "logits/rejected": -2.4576802253723145,
      "logps/chosen": -181.0323944091797,
      "logps/rejected": -226.46189880371094,
      "loss": 0.2441,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4320255517959595,
      "rewards/margins": 2.9019103050231934,
      "rewards/rejected": -4.333935737609863,
      "step": 1524
    },
    {
      "epoch": 2.0,
      "learning_rate": 4.5721407834317436e-05,
      "logits/chosen": -2.28127121925354,
      "logits/rejected": -2.286454677581787,
      "logps/chosen": -194.4881591796875,
      "logps/rejected": -241.7583465576172,
      "loss": 0.2224,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.432761549949646,
      "rewards/margins": 3.1377148628234863,
      "rewards/rejected": -4.570476531982422,
      "step": 1525
    },
    {
      "epoch": 2.0,
      "learning_rate": 4.571557844878271e-05,
      "logits/chosen": -1.9688351154327393,
      "logits/rejected": -2.005974292755127,
      "logps/chosen": -192.5892791748047,
      "logps/rejected": -242.64523315429688,
      "loss": 0.1622,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3402042388916016,
      "rewards/margins": 3.8366048336029053,
      "rewards/rejected": -5.1768083572387695,
      "step": 1526
    },
    {
      "epoch": 2.0,
      "learning_rate": 4.57097454669629e-05,
      "logits/chosen": -2.3066444396972656,
      "logits/rejected": -2.3153388500213623,
      "logps/chosen": -145.78077697753906,
      "logps/rejected": -192.21163940429688,
      "loss": 0.1604,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.429026484489441,
      "rewards/margins": 3.6721160411834717,
      "rewards/rejected": -5.101142406463623,
      "step": 1527
    },
    {
      "epoch": 2.0,
      "learning_rate": 4.5703908889870605e-05,
      "logits/chosen": -2.4281747341156006,
      "logits/rejected": -2.4992830753326416,
      "logps/chosen": -247.89735412597656,
      "logps/rejected": -306.4549560546875,
      "loss": 0.1279,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3649252653121948,
      "rewards/margins": 4.181218147277832,
      "rewards/rejected": -5.546143054962158,
      "step": 1528
    },
    {
      "epoch": 2.0,
      "learning_rate": 4.569806871851912e-05,
      "logits/chosen": -2.430555582046509,
      "logits/rejected": -2.3919808864593506,
      "logps/chosen": -304.0908508300781,
      "logps/rejected": -360.90771484375,
      "loss": 0.0538,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.44168150424957275,
      "rewards/margins": 5.871484279632568,
      "rewards/rejected": -6.313165664672852,
      "step": 1529
    },
    {
      "epoch": 2.0,
      "learning_rate": 4.5692224953922266e-05,
      "logits/chosen": -1.9216489791870117,
      "logits/rejected": -2.0158095359802246,
      "logps/chosen": -159.58489990234375,
      "logps/rejected": -228.5690460205078,
      "loss": 0.083,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9349645376205444,
      "rewards/margins": 5.259486675262451,
      "rewards/rejected": -6.194450855255127,
      "step": 1530
    },
    {
      "epoch": 2.0,
      "learning_rate": 4.5686377597094565e-05,
      "logits/chosen": -2.469989776611328,
      "logits/rejected": -2.4002585411071777,
      "logps/chosen": -270.3118896484375,
      "logps/rejected": -255.894287109375,
      "loss": 0.1311,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.131705641746521,
      "rewards/margins": 4.982357978820801,
      "rewards/rejected": -6.114063739776611,
      "step": 1531
    },
    {
      "epoch": 2.0,
      "learning_rate": 4.5680526649051134e-05,
      "logits/chosen": -2.191519260406494,
      "logits/rejected": -2.2068800926208496,
      "logps/chosen": -182.21990966796875,
      "logps/rejected": -229.68392944335938,
      "loss": 0.1128,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6737257838249207,
      "rewards/margins": 5.293555736541748,
      "rewards/rejected": -5.967280864715576,
      "step": 1532
    },
    {
      "epoch": 2.01,
      "learning_rate": 4.5674672110807713e-05,
      "logits/chosen": -2.2452383041381836,
      "logits/rejected": -2.2837133407592773,
      "logps/chosen": -190.92742919921875,
      "logps/rejected": -264.4825439453125,
      "loss": 0.0464,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.028515443205833435,
      "rewards/margins": 6.9200439453125,
      "rewards/rejected": -6.948559761047363,
      "step": 1533
    },
    {
      "epoch": 2.01,
      "learning_rate": 4.566881398338066e-05,
      "logits/chosen": -2.3002426624298096,
      "logits/rejected": -2.3245689868927,
      "logps/chosen": -150.93450927734375,
      "logps/rejected": -229.3854217529297,
      "loss": 0.0205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.013595461845398,
      "rewards/margins": 5.568351745605469,
      "rewards/rejected": -6.581946849822998,
      "step": 1534
    },
    {
      "epoch": 2.01,
      "learning_rate": 4.5662952267786974e-05,
      "logits/chosen": -2.2059741020202637,
      "logits/rejected": -2.230137825012207,
      "logps/chosen": -226.76358032226562,
      "logps/rejected": -293.0763854980469,
      "loss": 0.0507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1337634325027466,
      "rewards/margins": 6.117223739624023,
      "rewards/rejected": -7.250986576080322,
      "step": 1535
    },
    {
      "epoch": 2.01,
      "learning_rate": 4.565708696504425e-05,
      "logits/chosen": -2.300882577896118,
      "logits/rejected": -2.2846155166625977,
      "logps/chosen": -219.44903564453125,
      "logps/rejected": -288.78460693359375,
      "loss": 0.0896,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2709065675735474,
      "rewards/margins": 5.382192611694336,
      "rewards/rejected": -6.653099536895752,
      "step": 1536
    },
    {
      "epoch": 2.01,
      "learning_rate": 4.565121807617074e-05,
      "logits/chosen": -2.345656156539917,
      "logits/rejected": -2.3673782348632812,
      "logps/chosen": -218.91525268554688,
      "logps/rejected": -299.89044189453125,
      "loss": 0.0923,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8808358907699585,
      "rewards/margins": 5.190125465393066,
      "rewards/rejected": -6.070960998535156,
      "step": 1537
    },
    {
      "epoch": 2.01,
      "learning_rate": 4.5645345602185285e-05,
      "logits/chosen": -2.1208252906799316,
      "logits/rejected": -2.2008652687072754,
      "logps/chosen": -197.03170776367188,
      "logps/rejected": -291.86151123046875,
      "loss": 0.0368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3258799314498901,
      "rewards/margins": 6.69577693939209,
      "rewards/rejected": -8.02165699005127,
      "step": 1538
    },
    {
      "epoch": 2.01,
      "learning_rate": 4.563946954410738e-05,
      "logits/chosen": -2.2394111156463623,
      "logits/rejected": -2.2324044704437256,
      "logps/chosen": -207.92111206054688,
      "logps/rejected": -307.79339599609375,
      "loss": 0.0473,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1493899822235107,
      "rewards/margins": 6.962444305419922,
      "rewards/rejected": -8.111834526062012,
      "step": 1539
    },
    {
      "epoch": 2.02,
      "learning_rate": 4.563358990295711e-05,
      "logits/chosen": -2.33305287361145,
      "logits/rejected": -2.1754109859466553,
      "logps/chosen": -282.1322937011719,
      "logps/rejected": -337.01824951171875,
      "loss": 0.0921,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3482136726379395,
      "rewards/margins": 6.027097225189209,
      "rewards/rejected": -7.37531042098999,
      "step": 1540
    },
    {
      "epoch": 2.02,
      "learning_rate": 4.56277066797552e-05,
      "logits/chosen": -1.8929749727249146,
      "logits/rejected": -1.9339991807937622,
      "logps/chosen": -230.7352294921875,
      "logps/rejected": -318.597412109375,
      "loss": 0.0909,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.755754828453064,
      "rewards/margins": 6.48954439163208,
      "rewards/rejected": -8.245298385620117,
      "step": 1541
    },
    {
      "epoch": 2.02,
      "learning_rate": 4.562181987552301e-05,
      "logits/chosen": -2.1031947135925293,
      "logits/rejected": -2.1524558067321777,
      "logps/chosen": -184.82020568847656,
      "logps/rejected": -264.2920227050781,
      "loss": 0.0814,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6590917110443115,
      "rewards/margins": 7.221358299255371,
      "rewards/rejected": -7.8804497718811035,
      "step": 1542
    },
    {
      "epoch": 2.02,
      "learning_rate": 4.561592949128248e-05,
      "logits/chosen": -1.9674711227416992,
      "logits/rejected": -1.9534603357315063,
      "logps/chosen": -139.9609375,
      "logps/rejected": -188.78677368164062,
      "loss": 0.111,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1200040578842163,
      "rewards/margins": 5.612722396850586,
      "rewards/rejected": -6.73272705078125,
      "step": 1543
    },
    {
      "epoch": 2.02,
      "learning_rate": 4.561003552805623e-05,
      "logits/chosen": -2.174128770828247,
      "logits/rejected": -2.183529853820801,
      "logps/chosen": -185.51148986816406,
      "logps/rejected": -273.14007568359375,
      "loss": 0.016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1483352184295654,
      "rewards/margins": 7.032803058624268,
      "rewards/rejected": -9.181138038635254,
      "step": 1544
    },
    {
      "epoch": 2.02,
      "learning_rate": 4.560413798686745e-05,
      "logits/chosen": -2.0740859508514404,
      "logits/rejected": -2.122446060180664,
      "logps/chosen": -200.520263671875,
      "logps/rejected": -316.67706298828125,
      "loss": 0.0874,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.2280356884002686,
      "rewards/margins": 7.310361385345459,
      "rewards/rejected": -9.538396835327148,
      "step": 1545
    },
    {
      "epoch": 2.02,
      "learning_rate": 4.559823686873996e-05,
      "logits/chosen": -2.041712999343872,
      "logits/rejected": -2.1124374866485596,
      "logps/chosen": -183.34530639648438,
      "logps/rejected": -269.2987365722656,
      "loss": 0.0216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7693595886230469,
      "rewards/margins": 7.529676914215088,
      "rewards/rejected": -9.29903793334961,
      "step": 1546
    },
    {
      "epoch": 2.02,
      "learning_rate": 4.5592332174698236e-05,
      "logits/chosen": -1.954218864440918,
      "logits/rejected": -2.0636560916900635,
      "logps/chosen": -155.83811950683594,
      "logps/rejected": -233.9814910888672,
      "loss": 0.1034,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.49021577835083,
      "rewards/margins": 6.273719787597656,
      "rewards/rejected": -8.763935089111328,
      "step": 1547
    },
    {
      "epoch": 2.03,
      "learning_rate": 4.558642390576734e-05,
      "logits/chosen": -2.2698183059692383,
      "logits/rejected": -2.0585403442382812,
      "logps/chosen": -223.47071838378906,
      "logps/rejected": -262.3471374511719,
      "loss": 0.0279,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.512855052947998,
      "rewards/margins": 6.98197078704834,
      "rewards/rejected": -9.494826316833496,
      "step": 1548
    },
    {
      "epoch": 2.03,
      "learning_rate": 4.558051206297296e-05,
      "logits/chosen": -2.0435798168182373,
      "logits/rejected": -2.0743608474731445,
      "logps/chosen": -216.14540100097656,
      "logps/rejected": -329.14093017578125,
      "loss": 0.0452,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.84786319732666,
      "rewards/margins": 7.801297187805176,
      "rewards/rejected": -10.649162292480469,
      "step": 1549
    },
    {
      "epoch": 2.03,
      "learning_rate": 4.557459664734141e-05,
      "logits/chosen": -2.118642807006836,
      "logits/rejected": -2.2871828079223633,
      "logps/chosen": -207.33213806152344,
      "logps/rejected": -308.4573669433594,
      "loss": 0.0035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6876513957977295,
      "rewards/margins": 6.419556140899658,
      "rewards/rejected": -9.107207298278809,
      "step": 1550
    },
    {
      "epoch": 2.03,
      "learning_rate": 4.5568677659899625e-05,
      "logits/chosen": -1.952514886856079,
      "logits/rejected": -2.027898073196411,
      "logps/chosen": -198.62474060058594,
      "logps/rejected": -294.1047058105469,
      "loss": 0.0231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.833251953125,
      "rewards/margins": 7.694541931152344,
      "rewards/rejected": -10.52779483795166,
      "step": 1551
    },
    {
      "epoch": 2.03,
      "learning_rate": 4.556275510167517e-05,
      "logits/chosen": -2.3004631996154785,
      "logits/rejected": -2.2994089126586914,
      "logps/chosen": -228.01748657226562,
      "logps/rejected": -291.3690185546875,
      "loss": 0.0492,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.6044530868530273,
      "rewards/margins": 7.906800270080566,
      "rewards/rejected": -10.511253356933594,
      "step": 1552
    },
    {
      "epoch": 2.03,
      "learning_rate": 4.55568289736962e-05,
      "logits/chosen": -2.0798513889312744,
      "logits/rejected": -2.0810325145721436,
      "logps/chosen": -222.9461669921875,
      "logps/rejected": -292.3095397949219,
      "loss": 0.0485,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.89154052734375,
      "rewards/margins": 7.462136745452881,
      "rewards/rejected": -10.353677749633789,
      "step": 1553
    },
    {
      "epoch": 2.03,
      "learning_rate": 4.555089927699152e-05,
      "logits/chosen": -1.8029263019561768,
      "logits/rejected": -1.9061977863311768,
      "logps/chosen": -146.78433227539062,
      "logps/rejected": -235.67723083496094,
      "loss": 0.1222,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9760775566101074,
      "rewards/margins": 6.793957710266113,
      "rewards/rejected": -8.770035743713379,
      "step": 1554
    },
    {
      "epoch": 2.04,
      "learning_rate": 4.554496601259054e-05,
      "logits/chosen": -2.177738666534424,
      "logits/rejected": -2.196629524230957,
      "logps/chosen": -190.3204345703125,
      "logps/rejected": -254.96054077148438,
      "loss": 0.133,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.102139472961426,
      "rewards/margins": 5.640347480773926,
      "rewards/rejected": -8.742486953735352,
      "step": 1555
    },
    {
      "epoch": 2.04,
      "learning_rate": 4.553902918152329e-05,
      "logits/chosen": -1.7947801351547241,
      "logits/rejected": -1.947811484336853,
      "logps/chosen": -137.8947296142578,
      "logps/rejected": -254.4237823486328,
      "loss": 0.0797,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.180742025375366,
      "rewards/margins": 7.438215255737305,
      "rewards/rejected": -9.61895751953125,
      "step": 1556
    },
    {
      "epoch": 2.04,
      "learning_rate": 4.5533088784820424e-05,
      "logits/chosen": -2.155184030532837,
      "logits/rejected": -2.0984225273132324,
      "logps/chosen": -244.7468719482422,
      "logps/rejected": -332.7095947265625,
      "loss": 0.0025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4840188026428223,
      "rewards/margins": 7.49399995803833,
      "rewards/rejected": -10.978019714355469,
      "step": 1557
    },
    {
      "epoch": 2.04,
      "learning_rate": 4.55271448235132e-05,
      "logits/chosen": -2.0170555114746094,
      "logits/rejected": -2.008880615234375,
      "logps/chosen": -227.8103485107422,
      "logps/rejected": -325.2026062011719,
      "loss": 0.0275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.504916191101074,
      "rewards/margins": 9.110494613647461,
      "rewards/rejected": -12.615411758422852,
      "step": 1558
    },
    {
      "epoch": 2.04,
      "learning_rate": 4.552119729863353e-05,
      "logits/chosen": -1.8247603178024292,
      "logits/rejected": -1.9594895839691162,
      "logps/chosen": -191.88177490234375,
      "logps/rejected": -302.3210144042969,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6051971912384033,
      "rewards/margins": 8.71071720123291,
      "rewards/rejected": -12.315914154052734,
      "step": 1559
    },
    {
      "epoch": 2.04,
      "learning_rate": 4.5515246211213906e-05,
      "logits/chosen": -2.076730251312256,
      "logits/rejected": -2.1215121746063232,
      "logps/chosen": -228.9929962158203,
      "logps/rejected": -319.8890075683594,
      "loss": 0.0136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.890929698944092,
      "rewards/margins": 8.486113548278809,
      "rewards/rejected": -12.377043724060059,
      "step": 1560
    },
    {
      "epoch": 2.04,
      "learning_rate": 4.550929156228746e-05,
      "logits/chosen": -1.861020803451538,
      "logits/rejected": -1.8230705261230469,
      "logps/chosen": -229.69725036621094,
      "logps/rejected": -298.89349365234375,
      "loss": 0.0487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.429899215698242,
      "rewards/margins": 6.64678955078125,
      "rewards/rejected": -10.076687812805176,
      "step": 1561
    },
    {
      "epoch": 2.04,
      "learning_rate": 4.550333335288793e-05,
      "logits/chosen": -2.0620155334472656,
      "logits/rejected": -2.1032028198242188,
      "logps/chosen": -206.2206268310547,
      "logps/rejected": -270.24456787109375,
      "loss": 0.0924,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.5137698650360107,
      "rewards/margins": 6.249174118041992,
      "rewards/rejected": -9.762944221496582,
      "step": 1562
    },
    {
      "epoch": 2.05,
      "learning_rate": 4.5497371584049685e-05,
      "logits/chosen": -1.9463286399841309,
      "logits/rejected": -2.0142786502838135,
      "logps/chosen": -198.6859130859375,
      "logps/rejected": -268.861572265625,
      "loss": 0.0877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.667863368988037,
      "rewards/margins": 6.892669200897217,
      "rewards/rejected": -10.560531616210938,
      "step": 1563
    },
    {
      "epoch": 2.05,
      "learning_rate": 4.5491406256807704e-05,
      "logits/chosen": -1.9232243299484253,
      "logits/rejected": -2.064542055130005,
      "logps/chosen": -228.21578979492188,
      "logps/rejected": -361.20440673828125,
      "loss": 0.0362,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.642152309417725,
      "rewards/margins": 8.147041320800781,
      "rewards/rejected": -12.789194107055664,
      "step": 1564
    },
    {
      "epoch": 2.05,
      "learning_rate": 4.5485437372197584e-05,
      "logits/chosen": -1.951233148574829,
      "logits/rejected": -2.0665435791015625,
      "logps/chosen": -221.44845581054688,
      "logps/rejected": -282.0459289550781,
      "loss": 0.0875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.239589214324951,
      "rewards/margins": 7.598293304443359,
      "rewards/rejected": -10.837882041931152,
      "step": 1565
    },
    {
      "epoch": 2.05,
      "learning_rate": 4.5479464931255546e-05,
      "logits/chosen": -1.8419092893600464,
      "logits/rejected": -1.904127597808838,
      "logps/chosen": -164.92803955078125,
      "logps/rejected": -252.30120849609375,
      "loss": 0.0261,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.9283933639526367,
      "rewards/margins": 7.417722702026367,
      "rewards/rejected": -11.34611701965332,
      "step": 1566
    },
    {
      "epoch": 2.05,
      "learning_rate": 4.547348893501842e-05,
      "logits/chosen": -2.302267551422119,
      "logits/rejected": -2.3350188732147217,
      "logps/chosen": -292.8721618652344,
      "logps/rejected": -408.52679443359375,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.343358516693115,
      "rewards/margins": 8.37453556060791,
      "rewards/rejected": -12.717894554138184,
      "step": 1567
    },
    {
      "epoch": 2.05,
      "learning_rate": 4.5467509384523647e-05,
      "logits/chosen": -1.922215461730957,
      "logits/rejected": -1.9204285144805908,
      "logps/chosen": -255.0386505126953,
      "logps/rejected": -299.27880859375,
      "loss": 0.1312,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.7221174240112305,
      "rewards/margins": 6.29898738861084,
      "rewards/rejected": -11.02110481262207,
      "step": 1568
    },
    {
      "epoch": 2.05,
      "learning_rate": 4.5461526280809294e-05,
      "logits/chosen": -1.9684267044067383,
      "logits/rejected": -1.9687612056732178,
      "logps/chosen": -248.02020263671875,
      "logps/rejected": -312.8629455566406,
      "loss": 0.0442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.499403715133667,
      "rewards/margins": 8.902366638183594,
      "rewards/rejected": -12.401771545410156,
      "step": 1569
    },
    {
      "epoch": 2.05,
      "learning_rate": 4.5455539624914066e-05,
      "logits/chosen": -1.8702149391174316,
      "logits/rejected": -1.8160890340805054,
      "logps/chosen": -214.181640625,
      "logps/rejected": -280.1296081542969,
      "loss": 0.0828,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.006368160247803,
      "rewards/margins": 6.077723503112793,
      "rewards/rejected": -10.084092140197754,
      "step": 1570
    },
    {
      "epoch": 2.06,
      "learning_rate": 4.544954941787726e-05,
      "logits/chosen": -1.7309317588806152,
      "logits/rejected": -1.8509042263031006,
      "logps/chosen": -254.923828125,
      "logps/rejected": -311.92462158203125,
      "loss": 0.0631,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.9646990299224854,
      "rewards/margins": 7.722902297973633,
      "rewards/rejected": -11.687602043151855,
      "step": 1571
    },
    {
      "epoch": 2.06,
      "learning_rate": 4.5443555660738774e-05,
      "logits/chosen": -2.0469980239868164,
      "logits/rejected": -2.2485647201538086,
      "logps/chosen": -238.11585998535156,
      "logps/rejected": -320.4139099121094,
      "loss": 0.0535,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.201352119445801,
      "rewards/margins": 8.41773796081543,
      "rewards/rejected": -13.619089126586914,
      "step": 1572
    },
    {
      "epoch": 2.06,
      "learning_rate": 4.543755835453916e-05,
      "logits/chosen": -1.935351848602295,
      "logits/rejected": -1.998105525970459,
      "logps/chosen": -319.79559326171875,
      "logps/rejected": -425.8599853515625,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.961547374725342,
      "rewards/margins": 10.631295204162598,
      "rewards/rejected": -15.592842102050781,
      "step": 1573
    },
    {
      "epoch": 2.06,
      "learning_rate": 4.5431557500319556e-05,
      "logits/chosen": -2.1832823753356934,
      "logits/rejected": -2.1295218467712402,
      "logps/chosen": -223.51441955566406,
      "logps/rejected": -256.72711181640625,
      "loss": 0.1762,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.3360276222229,
      "rewards/margins": 6.326767444610596,
      "rewards/rejected": -10.662795066833496,
      "step": 1574
    },
    {
      "epoch": 2.06,
      "learning_rate": 4.542555309912173e-05,
      "logits/chosen": -1.9234460592269897,
      "logits/rejected": -2.1079142093658447,
      "logps/chosen": -220.8172607421875,
      "logps/rejected": -400.11480712890625,
      "loss": 0.0031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.353205680847168,
      "rewards/margins": 10.59358024597168,
      "rewards/rejected": -14.946784973144531,
      "step": 1575
    },
    {
      "epoch": 2.06,
      "learning_rate": 4.541954515198807e-05,
      "logits/chosen": -2.037827491760254,
      "logits/rejected": -2.095088481903076,
      "logps/chosen": -209.502685546875,
      "logps/rejected": -316.57293701171875,
      "loss": 0.1245,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.286071300506592,
      "rewards/margins": 7.779160499572754,
      "rewards/rejected": -12.065232276916504,
      "step": 1576
    },
    {
      "epoch": 2.06,
      "learning_rate": 4.541353365996157e-05,
      "logits/chosen": -1.951758623123169,
      "logits/rejected": -2.0389015674591064,
      "logps/chosen": -265.8280029296875,
      "logps/rejected": -333.02001953125,
      "loss": 0.0911,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.675872802734375,
      "rewards/margins": 7.314758777618408,
      "rewards/rejected": -11.990632057189941,
      "step": 1577
    },
    {
      "epoch": 2.07,
      "learning_rate": 4.540751862408585e-05,
      "logits/chosen": -1.473084568977356,
      "logits/rejected": -1.511156678199768,
      "logps/chosen": -172.9165496826172,
      "logps/rejected": -276.5952453613281,
      "loss": 0.1628,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.1226019859313965,
      "rewards/margins": 7.142666816711426,
      "rewards/rejected": -11.265267372131348,
      "step": 1578
    },
    {
      "epoch": 2.07,
      "learning_rate": 4.540150004540512e-05,
      "logits/chosen": -1.8454240560531616,
      "logits/rejected": -1.6991164684295654,
      "logps/chosen": -222.4569091796875,
      "logps/rejected": -299.02056884765625,
      "loss": 0.0387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.377161026000977,
      "rewards/margins": 8.041433334350586,
      "rewards/rejected": -12.418593406677246,
      "step": 1579
    },
    {
      "epoch": 2.07,
      "learning_rate": 4.539547792496424e-05,
      "logits/chosen": -2.0039446353912354,
      "logits/rejected": -2.05035662651062,
      "logps/chosen": -202.79391479492188,
      "logps/rejected": -289.2221984863281,
      "loss": 0.0882,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.039854526519775,
      "rewards/margins": 7.609333515167236,
      "rewards/rejected": -11.649188995361328,
      "step": 1580
    },
    {
      "epoch": 2.07,
      "learning_rate": 4.538945226380866e-05,
      "logits/chosen": -1.8065241575241089,
      "logits/rejected": -2.129643678665161,
      "logps/chosen": -215.80911254882812,
      "logps/rejected": -362.2320861816406,
      "loss": 0.0037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.694056272506714,
      "rewards/margins": 11.569538116455078,
      "rewards/rejected": -15.263594627380371,
      "step": 1581
    },
    {
      "epoch": 2.07,
      "learning_rate": 4.5383423062984455e-05,
      "logits/chosen": -1.9546061754226685,
      "logits/rejected": -2.007032632827759,
      "logps/chosen": -214.36041259765625,
      "logps/rejected": -333.48345947265625,
      "loss": 0.0447,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.047149181365967,
      "rewards/margins": 8.15298843383789,
      "rewards/rejected": -12.200136184692383,
      "step": 1582
    },
    {
      "epoch": 2.07,
      "learning_rate": 4.5377390323538305e-05,
      "logits/chosen": -2.035081148147583,
      "logits/rejected": -2.1894919872283936,
      "logps/chosen": -214.34088134765625,
      "logps/rejected": -353.6617736816406,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.582559108734131,
      "rewards/margins": 9.10820198059082,
      "rewards/rejected": -13.690760612487793,
      "step": 1583
    },
    {
      "epoch": 2.07,
      "learning_rate": 4.5371354046517516e-05,
      "logits/chosen": -1.7794345617294312,
      "logits/rejected": -1.7997311353683472,
      "logps/chosen": -249.14747619628906,
      "logps/rejected": -335.20977783203125,
      "loss": 0.0029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.026828289031982,
      "rewards/margins": 8.605960845947266,
      "rewards/rejected": -13.632789611816406,
      "step": 1584
    },
    {
      "epoch": 2.07,
      "learning_rate": 4.536531423297001e-05,
      "logits/chosen": -2.006917953491211,
      "logits/rejected": -2.064652681350708,
      "logps/chosen": -224.6466064453125,
      "logps/rejected": -331.7161865234375,
      "loss": 0.042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.988997459411621,
      "rewards/margins": 8.227411270141602,
      "rewards/rejected": -13.216409683227539,
      "step": 1585
    },
    {
      "epoch": 2.08,
      "learning_rate": 4.535927088394431e-05,
      "logits/chosen": -1.6211562156677246,
      "logits/rejected": -1.7851699590682983,
      "logps/chosen": -235.837890625,
      "logps/rejected": -384.5337829589844,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.735535621643066,
      "rewards/margins": 7.668628215789795,
      "rewards/rejected": -13.404163360595703,
      "step": 1586
    },
    {
      "epoch": 2.08,
      "learning_rate": 4.5353224000489555e-05,
      "logits/chosen": -2.165318250656128,
      "logits/rejected": -2.2333309650421143,
      "logps/chosen": -211.68421936035156,
      "logps/rejected": -291.989501953125,
      "loss": 0.088,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.371261119842529,
      "rewards/margins": 7.36502742767334,
      "rewards/rejected": -11.736289024353027,
      "step": 1587
    },
    {
      "epoch": 2.08,
      "learning_rate": 4.534717358365551e-05,
      "logits/chosen": -1.9365509748458862,
      "logits/rejected": -1.9755022525787354,
      "logps/chosen": -214.47874450683594,
      "logps/rejected": -273.87493896484375,
      "loss": 0.0655,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.31796932220459,
      "rewards/margins": 5.916989803314209,
      "rewards/rejected": -10.234959602355957,
      "step": 1588
    },
    {
      "epoch": 2.08,
      "learning_rate": 4.534111963449254e-05,
      "logits/chosen": -1.9963104724884033,
      "logits/rejected": -2.051135778427124,
      "logps/chosen": -246.76168823242188,
      "logps/rejected": -350.1937561035156,
      "loss": 0.0887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.5803420543670654,
      "rewards/margins": 8.423789978027344,
      "rewards/rejected": -12.004133224487305,
      "step": 1589
    },
    {
      "epoch": 2.08,
      "learning_rate": 4.5335062154051625e-05,
      "logits/chosen": -2.1684515476226807,
      "logits/rejected": -2.0885422229766846,
      "logps/chosen": -198.07298278808594,
      "logps/rejected": -275.5907287597656,
      "loss": 0.0503,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.1592936515808105,
      "rewards/margins": 7.279091835021973,
      "rewards/rejected": -10.438385009765625,
      "step": 1590
    },
    {
      "epoch": 2.08,
      "learning_rate": 4.532900114338437e-05,
      "logits/chosen": -1.7232838869094849,
      "logits/rejected": -1.7098734378814697,
      "logps/chosen": -180.58285522460938,
      "logps/rejected": -214.90383911132812,
      "loss": 0.1814,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.376774787902832,
      "rewards/margins": 5.187775611877441,
      "rewards/rejected": -8.56455135345459,
      "step": 1591
    },
    {
      "epoch": 2.08,
      "learning_rate": 4.5322936603542984e-05,
      "logits/chosen": -2.0076513290405273,
      "logits/rejected": -2.0869903564453125,
      "logps/chosen": -233.26451110839844,
      "logps/rejected": -340.70947265625,
      "loss": 0.0886,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.3201494216918945,
      "rewards/margins": 6.867827415466309,
      "rewards/rejected": -11.187976837158203,
      "step": 1592
    },
    {
      "epoch": 2.08,
      "learning_rate": 4.531686853558028e-05,
      "logits/chosen": -2.168454170227051,
      "logits/rejected": -2.149108648300171,
      "logps/chosen": -235.9113311767578,
      "logps/rejected": -306.544677734375,
      "loss": 0.1488,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.6051135063171387,
      "rewards/margins": 6.618370056152344,
      "rewards/rejected": -10.223483085632324,
      "step": 1593
    },
    {
      "epoch": 2.09,
      "learning_rate": 4.53107969405497e-05,
      "logits/chosen": -2.172264575958252,
      "logits/rejected": -2.203526735305786,
      "logps/chosen": -381.8232116699219,
      "logps/rejected": -438.64007568359375,
      "loss": 0.1118,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.092607021331787,
      "rewards/margins": 7.635544300079346,
      "rewards/rejected": -11.728151321411133,
      "step": 1594
    },
    {
      "epoch": 2.09,
      "learning_rate": 4.530472181950528e-05,
      "logits/chosen": -2.014963388442993,
      "logits/rejected": -1.9751347303390503,
      "logps/chosen": -268.2021179199219,
      "logps/rejected": -323.82379150390625,
      "loss": 0.0456,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.099268436431885,
      "rewards/margins": 8.01276683807373,
      "rewards/rejected": -12.112035751342773,
      "step": 1595
    },
    {
      "epoch": 2.09,
      "learning_rate": 4.5298643173501693e-05,
      "logits/chosen": -2.157127618789673,
      "logits/rejected": -2.2155063152313232,
      "logps/chosen": -326.23492431640625,
      "logps/rejected": -361.61077880859375,
      "loss": 0.063,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.8939595222473145,
      "rewards/margins": 7.392580032348633,
      "rewards/rejected": -11.286539077758789,
      "step": 1596
    },
    {
      "epoch": 2.09,
      "learning_rate": 4.5292561003594205e-05,
      "logits/chosen": -1.866766333580017,
      "logits/rejected": -1.8167577981948853,
      "logps/chosen": -219.24777221679688,
      "logps/rejected": -265.5517578125,
      "loss": 0.1723,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.417174816131592,
      "rewards/margins": 5.938780307769775,
      "rewards/rejected": -10.355955123901367,
      "step": 1597
    },
    {
      "epoch": 2.09,
      "learning_rate": 4.5286475310838695e-05,
      "logits/chosen": -2.137641191482544,
      "logits/rejected": -2.1744580268859863,
      "logps/chosen": -211.7652587890625,
      "logps/rejected": -323.32415771484375,
      "loss": 0.0537,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.528270959854126,
      "rewards/margins": 9.253046989440918,
      "rewards/rejected": -12.781318664550781,
      "step": 1598
    },
    {
      "epoch": 2.09,
      "learning_rate": 4.5280386096291664e-05,
      "logits/chosen": -2.074810266494751,
      "logits/rejected": -2.0623838901519775,
      "logps/chosen": -218.1590576171875,
      "logps/rejected": -339.4794006347656,
      "loss": 0.044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1840147972106934,
      "rewards/margins": 8.886053085327148,
      "rewards/rejected": -12.070067405700684,
      "step": 1599
    },
    {
      "epoch": 2.09,
      "learning_rate": 4.52742933610102e-05,
      "logits/chosen": -1.9109052419662476,
      "logits/rejected": -1.8507012128829956,
      "logps/chosen": -220.08070373535156,
      "logps/rejected": -284.7148742675781,
      "loss": 0.0936,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.1194047927856445,
      "rewards/margins": 7.436119079589844,
      "rewards/rejected": -10.555523872375488,
      "step": 1600
    },
    {
      "epoch": 2.1,
      "learning_rate": 4.526819710605205e-05,
      "logits/chosen": -2.1238362789154053,
      "logits/rejected": -2.0772507190704346,
      "logps/chosen": -201.04638671875,
      "logps/rejected": -280.64678955078125,
      "loss": 0.1025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.489070415496826,
      "rewards/margins": 6.871294975280762,
      "rewards/rejected": -10.360365867614746,
      "step": 1601
    },
    {
      "epoch": 2.1,
      "learning_rate": 4.526209733247551e-05,
      "logits/chosen": -1.8819748163223267,
      "logits/rejected": -1.8802152872085571,
      "logps/chosen": -240.85882568359375,
      "logps/rejected": -351.0258483886719,
      "loss": 0.0314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.410835266113281,
      "rewards/margins": 7.961416721343994,
      "rewards/rejected": -12.372251510620117,
      "step": 1602
    },
    {
      "epoch": 2.1,
      "learning_rate": 4.5255994041339536e-05,
      "logits/chosen": -2.0957300662994385,
      "logits/rejected": -2.22131609916687,
      "logps/chosen": -240.03170776367188,
      "logps/rejected": -405.8812255859375,
      "loss": 0.0495,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5040841102600098,
      "rewards/margins": 10.249372482299805,
      "rewards/rejected": -13.75345516204834,
      "step": 1603
    },
    {
      "epoch": 2.1,
      "learning_rate": 4.524988723370367e-05,
      "logits/chosen": -1.8505096435546875,
      "logits/rejected": -1.8317203521728516,
      "logps/chosen": -206.16671752929688,
      "logps/rejected": -289.6446838378906,
      "loss": 0.0843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2112929821014404,
      "rewards/margins": 6.922733306884766,
      "rewards/rejected": -10.134026527404785,
      "step": 1604
    },
    {
      "epoch": 2.1,
      "learning_rate": 4.5243776910628076e-05,
      "logits/chosen": -1.963594913482666,
      "logits/rejected": -2.020098924636841,
      "logps/chosen": -210.90228271484375,
      "logps/rejected": -305.5580749511719,
      "loss": 0.0871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.103038787841797,
      "rewards/margins": 7.604170322418213,
      "rewards/rejected": -10.707208633422852,
      "step": 1605
    },
    {
      "epoch": 2.1,
      "learning_rate": 4.523766307317353e-05,
      "logits/chosen": -1.9862631559371948,
      "logits/rejected": -2.030336380004883,
      "logps/chosen": -219.91226196289062,
      "logps/rejected": -288.57550048828125,
      "loss": 0.0044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6835989952087402,
      "rewards/margins": 7.093358993530273,
      "rewards/rejected": -10.776958465576172,
      "step": 1606
    },
    {
      "epoch": 2.1,
      "learning_rate": 4.5231545722401406e-05,
      "logits/chosen": -2.2059803009033203,
      "logits/rejected": -2.218961477279663,
      "logps/chosen": -197.95693969726562,
      "logps/rejected": -261.8663024902344,
      "loss": 0.0883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.4130430221557617,
      "rewards/margins": 7.487000465393066,
      "rewards/rejected": -9.900043487548828,
      "step": 1607
    },
    {
      "epoch": 2.1,
      "learning_rate": 4.522542485937369e-05,
      "logits/chosen": -2.0145654678344727,
      "logits/rejected": -2.118668794631958,
      "logps/chosen": -221.60939025878906,
      "logps/rejected": -365.8251647949219,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.232522487640381,
      "rewards/margins": 9.566020965576172,
      "rewards/rejected": -12.798542976379395,
      "step": 1608
    },
    {
      "epoch": 2.11,
      "learning_rate": 4.5219300485152983e-05,
      "logits/chosen": -2.0122945308685303,
      "logits/rejected": -2.0334525108337402,
      "logps/chosen": -199.4896697998047,
      "logps/rejected": -276.1574401855469,
      "loss": 0.0618,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5203516483306885,
      "rewards/margins": 7.354372978210449,
      "rewards/rejected": -10.874725341796875,
      "step": 1609
    },
    {
      "epoch": 2.11,
      "learning_rate": 4.52131726008025e-05,
      "logits/chosen": -1.971169114112854,
      "logits/rejected": -2.1381020545959473,
      "logps/chosen": -266.3401184082031,
      "logps/rejected": -325.87091064453125,
      "loss": 0.0734,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.6472837924957275,
      "rewards/margins": 7.2722978591918945,
      "rewards/rejected": -10.91958236694336,
      "step": 1610
    },
    {
      "epoch": 2.11,
      "learning_rate": 4.520704120738606e-05,
      "logits/chosen": -1.8407872915267944,
      "logits/rejected": -1.9220272302627563,
      "logps/chosen": -185.66297912597656,
      "logps/rejected": -286.11334228515625,
      "loss": 0.0543,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.526944160461426,
      "rewards/margins": 6.661890506744385,
      "rewards/rejected": -10.188834190368652,
      "step": 1611
    },
    {
      "epoch": 2.11,
      "learning_rate": 4.520090630596808e-05,
      "logits/chosen": -1.508141040802002,
      "logits/rejected": -1.6697499752044678,
      "logps/chosen": -205.7053680419922,
      "logps/rejected": -327.8975830078125,
      "loss": 0.0521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4045472145080566,
      "rewards/margins": 7.769204139709473,
      "rewards/rejected": -11.173750877380371,
      "step": 1612
    },
    {
      "epoch": 2.11,
      "learning_rate": 4.519476789761361e-05,
      "logits/chosen": -2.0378808975219727,
      "logits/rejected": -2.055891990661621,
      "logps/chosen": -224.57933044433594,
      "logps/rejected": -330.0787658691406,
      "loss": 0.0511,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.4485056400299072,
      "rewards/margins": 7.820329666137695,
      "rewards/rejected": -11.268835067749023,
      "step": 1613
    },
    {
      "epoch": 2.11,
      "learning_rate": 4.5188625983388275e-05,
      "logits/chosen": -1.7743136882781982,
      "logits/rejected": -1.8478460311889648,
      "logps/chosen": -247.61187744140625,
      "logps/rejected": -324.42425537109375,
      "loss": 0.1324,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.284206390380859,
      "rewards/margins": 6.411763668060303,
      "rewards/rejected": -10.69597053527832,
      "step": 1614
    },
    {
      "epoch": 2.11,
      "learning_rate": 4.5182480564358356e-05,
      "logits/chosen": -1.9719691276550293,
      "logits/rejected": -2.0874102115631104,
      "logps/chosen": -214.29307556152344,
      "logps/rejected": -352.97418212890625,
      "loss": 0.0874,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.3708295822143555,
      "rewards/margins": 9.124308586120605,
      "rewards/rejected": -11.495138168334961,
      "step": 1615
    },
    {
      "epoch": 2.11,
      "learning_rate": 4.517633164159071e-05,
      "logits/chosen": -2.063558578491211,
      "logits/rejected": -2.044651508331299,
      "logps/chosen": -250.53836059570312,
      "logps/rejected": -333.0967102050781,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.142364978790283,
      "rewards/margins": 7.67378568649292,
      "rewards/rejected": -11.816149711608887,
      "step": 1616
    },
    {
      "epoch": 2.12,
      "learning_rate": 4.5170179216152785e-05,
      "logits/chosen": -2.05293607711792,
      "logits/rejected": -2.0275938510894775,
      "logps/chosen": -251.1569061279297,
      "logps/rejected": -274.8096923828125,
      "loss": 0.0885,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5648229122161865,
      "rewards/margins": 6.097499370574951,
      "rewards/rejected": -9.662322044372559,
      "step": 1617
    },
    {
      "epoch": 2.12,
      "learning_rate": 4.516402328911269e-05,
      "logits/chosen": -2.0880846977233887,
      "logits/rejected": -2.10282301902771,
      "logps/chosen": -210.91859436035156,
      "logps/rejected": -306.0312805175781,
      "loss": 0.0452,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1213300228118896,
      "rewards/margins": 8.73926830291748,
      "rewards/rejected": -11.86059856414795,
      "step": 1618
    },
    {
      "epoch": 2.12,
      "learning_rate": 4.5157863861539093e-05,
      "logits/chosen": -2.01450777053833,
      "logits/rejected": -1.9900140762329102,
      "logps/chosen": -247.73287963867188,
      "logps/rejected": -339.9057312011719,
      "loss": 0.0035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3483333587646484,
      "rewards/margins": 9.037501335144043,
      "rewards/rejected": -12.385833740234375,
      "step": 1619
    },
    {
      "epoch": 2.12,
      "learning_rate": 4.515170093450129e-05,
      "logits/chosen": -1.9653971195220947,
      "logits/rejected": -2.040536642074585,
      "logps/chosen": -239.28778076171875,
      "logps/rejected": -357.8448486328125,
      "loss": 0.017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.8111658096313477,
      "rewards/margins": 8.509544372558594,
      "rewards/rejected": -12.320709228515625,
      "step": 1620
    },
    {
      "epoch": 2.12,
      "learning_rate": 4.5145534509069196e-05,
      "logits/chosen": -1.8105957508087158,
      "logits/rejected": -1.7471611499786377,
      "logps/chosen": -196.7708740234375,
      "logps/rejected": -298.52545166015625,
      "loss": 0.046,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.7363200187683105,
      "rewards/margins": 7.920873641967773,
      "rewards/rejected": -10.657193183898926,
      "step": 1621
    },
    {
      "epoch": 2.12,
      "learning_rate": 4.513936458631332e-05,
      "logits/chosen": -2.146426200866699,
      "logits/rejected": -2.1980879306793213,
      "logps/chosen": -262.87750244140625,
      "logps/rejected": -332.66558837890625,
      "loss": 0.0117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.306097984313965,
      "rewards/margins": 8.095102310180664,
      "rewards/rejected": -12.401199340820312,
      "step": 1622
    },
    {
      "epoch": 2.12,
      "learning_rate": 4.5133191167304754e-05,
      "logits/chosen": -1.9258334636688232,
      "logits/rejected": -1.9266880750656128,
      "logps/chosen": -192.65664672851562,
      "logps/rejected": -287.0689697265625,
      "loss": 0.0043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7276978492736816,
      "rewards/margins": 8.794516563415527,
      "rewards/rejected": -11.522215843200684,
      "step": 1623
    },
    {
      "epoch": 2.13,
      "learning_rate": 4.5127014253115246e-05,
      "logits/chosen": -1.9906028509140015,
      "logits/rejected": -2.0530624389648438,
      "logps/chosen": -270.1056213378906,
      "logps/rejected": -329.58441162109375,
      "loss": 0.0439,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.131108283996582,
      "rewards/margins": 8.448394775390625,
      "rewards/rejected": -12.579503059387207,
      "step": 1624
    },
    {
      "epoch": 2.13,
      "learning_rate": 4.5120833844817116e-05,
      "logits/chosen": -1.2688982486724854,
      "logits/rejected": -1.2891889810562134,
      "logps/chosen": -228.94691467285156,
      "logps/rejected": -296.3216552734375,
      "loss": 0.0514,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.573295593261719,
      "rewards/margins": 6.978508472442627,
      "rewards/rejected": -11.551802635192871,
      "step": 1625
    },
    {
      "epoch": 2.13,
      "learning_rate": 4.5114649943483306e-05,
      "logits/chosen": -1.7697856426239014,
      "logits/rejected": -1.8215487003326416,
      "logps/chosen": -194.67367553710938,
      "logps/rejected": -292.50726318359375,
      "loss": 0.1312,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.337482213973999,
      "rewards/margins": 6.588993072509766,
      "rewards/rejected": -9.926475524902344,
      "step": 1626
    },
    {
      "epoch": 2.13,
      "learning_rate": 4.510846255018736e-05,
      "logits/chosen": -2.0337822437286377,
      "logits/rejected": -2.033005714416504,
      "logps/chosen": -247.71368408203125,
      "logps/rejected": -357.8754577636719,
      "loss": 0.023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.704446315765381,
      "rewards/margins": 8.37730884552002,
      "rewards/rejected": -13.081753730773926,
      "step": 1627
    },
    {
      "epoch": 2.13,
      "learning_rate": 4.5102271666003417e-05,
      "logits/chosen": -1.8250856399536133,
      "logits/rejected": -1.890662670135498,
      "logps/chosen": -309.4023132324219,
      "logps/rejected": -368.8353576660156,
      "loss": 0.0464,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.654391288757324,
      "rewards/margins": 7.280395984649658,
      "rewards/rejected": -11.93478775024414,
      "step": 1628
    },
    {
      "epoch": 2.13,
      "learning_rate": 4.509607729200625e-05,
      "logits/chosen": -1.5864503383636475,
      "logits/rejected": -1.6689658164978027,
      "logps/chosen": -224.78378295898438,
      "logps/rejected": -361.28692626953125,
      "loss": 0.0471,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.2646989822387695,
      "rewards/margins": 9.243759155273438,
      "rewards/rejected": -12.508459091186523,
      "step": 1629
    },
    {
      "epoch": 2.13,
      "learning_rate": 4.50898794292712e-05,
      "logits/chosen": -1.9407392740249634,
      "logits/rejected": -1.9865281581878662,
      "logps/chosen": -327.77001953125,
      "logps/rejected": -436.6799011230469,
      "loss": 0.0749,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.880475044250488,
      "rewards/margins": 8.208063125610352,
      "rewards/rejected": -14.08853816986084,
      "step": 1630
    },
    {
      "epoch": 2.13,
      "learning_rate": 4.508367807887425e-05,
      "logits/chosen": -1.7678276300430298,
      "logits/rejected": -1.7630263566970825,
      "logps/chosen": -344.8454284667969,
      "logps/rejected": -394.4046325683594,
      "loss": 0.1963,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.344852924346924,
      "rewards/margins": 6.4874467849731445,
      "rewards/rejected": -12.83229923248291,
      "step": 1631
    },
    {
      "epoch": 2.14,
      "learning_rate": 4.5077473241891965e-05,
      "logits/chosen": -1.511955738067627,
      "logits/rejected": -1.4905922412872314,
      "logps/chosen": -220.30099487304688,
      "logps/rejected": -281.9308776855469,
      "loss": 0.2282,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.048691272735596,
      "rewards/margins": 5.856573581695557,
      "rewards/rejected": -9.905265808105469,
      "step": 1632
    },
    {
      "epoch": 2.14,
      "learning_rate": 4.507126491940153e-05,
      "logits/chosen": -1.8332291841506958,
      "logits/rejected": -1.8841722011566162,
      "logps/chosen": -225.41574096679688,
      "logps/rejected": -307.5355529785156,
      "loss": 0.0921,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.2159814834594727,
      "rewards/margins": 7.965794086456299,
      "rewards/rejected": -11.181775093078613,
      "step": 1633
    },
    {
      "epoch": 2.14,
      "learning_rate": 4.506505311248072e-05,
      "logits/chosen": -1.708308219909668,
      "logits/rejected": -1.7283846139907837,
      "logps/chosen": -204.90386962890625,
      "logps/rejected": -330.06927490234375,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.275350570678711,
      "rewards/margins": 9.99798583984375,
      "rewards/rejected": -14.273336410522461,
      "step": 1634
    },
    {
      "epoch": 2.14,
      "learning_rate": 4.505883782220793e-05,
      "logits/chosen": -1.7285094261169434,
      "logits/rejected": -1.7134628295898438,
      "logps/chosen": -236.44061279296875,
      "logps/rejected": -302.3081359863281,
      "loss": 0.1065,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.248753547668457,
      "rewards/margins": 6.385387420654297,
      "rewards/rejected": -11.634140014648438,
      "step": 1635
    },
    {
      "epoch": 2.14,
      "learning_rate": 4.505261904966214e-05,
      "logits/chosen": -1.644340991973877,
      "logits/rejected": -1.7456663846969604,
      "logps/chosen": -234.6942138671875,
      "logps/rejected": -330.1268615722656,
      "loss": 0.0055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.138976573944092,
      "rewards/margins": 7.677927017211914,
      "rewards/rejected": -12.816903114318848,
      "step": 1636
    },
    {
      "epoch": 2.14,
      "learning_rate": 4.5046396795922964e-05,
      "logits/chosen": -1.590599775314331,
      "logits/rejected": -1.6563043594360352,
      "logps/chosen": -243.24896240234375,
      "logps/rejected": -286.80499267578125,
      "loss": 0.0517,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.77515983581543,
      "rewards/margins": 6.981686115264893,
      "rewards/rejected": -11.75684642791748,
      "step": 1637
    },
    {
      "epoch": 2.14,
      "learning_rate": 4.504017106207059e-05,
      "logits/chosen": -1.8463704586029053,
      "logits/rejected": -1.9400237798690796,
      "logps/chosen": -263.376953125,
      "logps/rejected": -324.1238708496094,
      "loss": 0.0479,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.044637203216553,
      "rewards/margins": 8.752671241760254,
      "rewards/rejected": -13.797307968139648,
      "step": 1638
    },
    {
      "epoch": 2.14,
      "learning_rate": 4.5033941849185835e-05,
      "logits/chosen": -2.0151426792144775,
      "logits/rejected": -2.0619280338287354,
      "logps/chosen": -239.4160919189453,
      "logps/rejected": -311.9981384277344,
      "loss": 0.0459,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.772345542907715,
      "rewards/margins": 7.730282783508301,
      "rewards/rejected": -12.502628326416016,
      "step": 1639
    },
    {
      "epoch": 2.15,
      "learning_rate": 4.50277091583501e-05,
      "logits/chosen": -1.5663697719573975,
      "logits/rejected": -1.6064140796661377,
      "logps/chosen": -182.79518127441406,
      "logps/rejected": -268.147216796875,
      "loss": 0.1124,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.8507080078125,
      "rewards/margins": 7.0118231773376465,
      "rewards/rejected": -10.862531661987305,
      "step": 1640
    },
    {
      "epoch": 2.15,
      "learning_rate": 4.5021472990645404e-05,
      "logits/chosen": -1.7681742906570435,
      "logits/rejected": -1.8470631837844849,
      "logps/chosen": -315.16131591796875,
      "logps/rejected": -430.01513671875,
      "loss": 0.088,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.021973133087158,
      "rewards/margins": 8.149192810058594,
      "rewards/rejected": -13.171164512634277,
      "step": 1641
    },
    {
      "epoch": 2.15,
      "learning_rate": 4.501523334715436e-05,
      "logits/chosen": -1.843216896057129,
      "logits/rejected": -1.9197378158569336,
      "logps/chosen": -242.83187866210938,
      "logps/rejected": -410.14581298828125,
      "loss": 0.0874,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.327381610870361,
      "rewards/margins": 8.265216827392578,
      "rewards/rejected": -12.592597961425781,
      "step": 1642
    },
    {
      "epoch": 2.15,
      "learning_rate": 4.5008990228960176e-05,
      "logits/chosen": -1.5981818437576294,
      "logits/rejected": -1.7583653926849365,
      "logps/chosen": -209.43771362304688,
      "logps/rejected": -317.85296630859375,
      "loss": 0.1179,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.252575397491455,
      "rewards/margins": 8.589457511901855,
      "rewards/rejected": -12.842031478881836,
      "step": 1643
    },
    {
      "epoch": 2.15,
      "learning_rate": 4.500274363714671e-05,
      "logits/chosen": -1.8431150913238525,
      "logits/rejected": -2.0487449169158936,
      "logps/chosen": -255.32421875,
      "logps/rejected": -387.9079895019531,
      "loss": 0.0803,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.5627946853637695,
      "rewards/margins": 8.269213676452637,
      "rewards/rejected": -12.832008361816406,
      "step": 1644
    },
    {
      "epoch": 2.15,
      "learning_rate": 4.4996493572798356e-05,
      "logits/chosen": -2.1761112213134766,
      "logits/rejected": -2.2496793270111084,
      "logps/chosen": -270.3349914550781,
      "logps/rejected": -391.78509521484375,
      "loss": 0.087,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.312464714050293,
      "rewards/margins": 7.775787353515625,
      "rewards/rejected": -12.088252067565918,
      "step": 1645
    },
    {
      "epoch": 2.15,
      "learning_rate": 4.499024003700015e-05,
      "logits/chosen": -1.5933220386505127,
      "logits/rejected": -1.6437098979949951,
      "logps/chosen": -202.7498016357422,
      "logps/rejected": -268.6524353027344,
      "loss": 0.102,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.125113010406494,
      "rewards/margins": 7.220837116241455,
      "rewards/rejected": -10.34595012664795,
      "step": 1646
    },
    {
      "epoch": 2.16,
      "learning_rate": 4.498398303083774e-05,
      "logits/chosen": -1.565222144126892,
      "logits/rejected": -1.5468480587005615,
      "logps/chosen": -231.1023712158203,
      "logps/rejected": -317.9393005371094,
      "loss": 0.0982,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.608382701873779,
      "rewards/margins": 6.992830276489258,
      "rewards/rejected": -11.601211547851562,
      "step": 1647
    },
    {
      "epoch": 2.16,
      "learning_rate": 4.497772255539735e-05,
      "logits/chosen": -1.8142828941345215,
      "logits/rejected": -1.849523901939392,
      "logps/chosen": -186.3400115966797,
      "logps/rejected": -267.7499084472656,
      "loss": 0.064,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.7400169372558594,
      "rewards/margins": 6.862453937530518,
      "rewards/rejected": -10.602472305297852,
      "step": 1648
    },
    {
      "epoch": 2.16,
      "learning_rate": 4.497145861176581e-05,
      "logits/chosen": -1.8521313667297363,
      "logits/rejected": -1.8114895820617676,
      "logps/chosen": -211.46078491210938,
      "logps/rejected": -263.2080993652344,
      "loss": 0.0491,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.9873476028442383,
      "rewards/margins": 6.93930721282959,
      "rewards/rejected": -10.926654815673828,
      "step": 1649
    },
    {
      "epoch": 2.16,
      "learning_rate": 4.496519120103057e-05,
      "logits/chosen": -1.874268889427185,
      "logits/rejected": -2.000852584838867,
      "logps/chosen": -219.22030639648438,
      "logps/rejected": -360.3320617675781,
      "loss": 0.0694,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.9966840744018555,
      "rewards/margins": 7.829019546508789,
      "rewards/rejected": -11.825703620910645,
      "step": 1650
    },
    {
      "epoch": 2.16,
      "learning_rate": 4.495892032427966e-05,
      "logits/chosen": -2.003627061843872,
      "logits/rejected": -2.088552236557007,
      "logps/chosen": -213.91397094726562,
      "logps/rejected": -319.4725646972656,
      "loss": 0.0451,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.4441897869110107,
      "rewards/margins": 8.1718111038208,
      "rewards/rejected": -11.61600112915039,
      "step": 1651
    },
    {
      "epoch": 2.16,
      "learning_rate": 4.495264598260174e-05,
      "logits/chosen": -1.4233427047729492,
      "logits/rejected": -1.4113380908966064,
      "logps/chosen": -239.0942840576172,
      "logps/rejected": -325.3289489746094,
      "loss": 0.1248,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.8961760997772217,
      "rewards/margins": 7.048993110656738,
      "rewards/rejected": -10.945169448852539,
      "step": 1652
    },
    {
      "epoch": 2.16,
      "learning_rate": 4.4946368177086034e-05,
      "logits/chosen": -1.8109068870544434,
      "logits/rejected": -1.8243604898452759,
      "logps/chosen": -223.3125,
      "logps/rejected": -280.24566650390625,
      "loss": 0.0913,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.3544464111328125,
      "rewards/margins": 7.040537357330322,
      "rewards/rejected": -10.394983291625977,
      "step": 1653
    },
    {
      "epoch": 2.16,
      "learning_rate": 4.49400869088224e-05,
      "logits/chosen": -1.8591406345367432,
      "logits/rejected": -1.917752742767334,
      "logps/chosen": -265.19586181640625,
      "logps/rejected": -344.5182800292969,
      "loss": 0.0802,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.180366039276123,
      "rewards/margins": 7.217251300811768,
      "rewards/rejected": -11.39761734008789,
      "step": 1654
    },
    {
      "epoch": 2.17,
      "learning_rate": 4.493380217890128e-05,
      "logits/chosen": -2.016444683074951,
      "logits/rejected": -2.066431999206543,
      "logps/chosen": -249.1741180419922,
      "logps/rejected": -367.8361511230469,
      "loss": 0.0872,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.942394733428955,
      "rewards/margins": 7.774467468261719,
      "rewards/rejected": -11.716863632202148,
      "step": 1655
    },
    {
      "epoch": 2.17,
      "learning_rate": 4.492751398841372e-05,
      "logits/chosen": -1.8434172868728638,
      "logits/rejected": -1.943632960319519,
      "logps/chosen": -183.25062561035156,
      "logps/rejected": -272.4522705078125,
      "loss": 0.0879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.8946924209594727,
      "rewards/margins": 7.5939788818359375,
      "rewards/rejected": -11.488670349121094,
      "step": 1656
    },
    {
      "epoch": 2.17,
      "learning_rate": 4.492122233845138e-05,
      "logits/chosen": -1.9134418964385986,
      "logits/rejected": -1.9178599119186401,
      "logps/chosen": -230.68223571777344,
      "logps/rejected": -401.3302307128906,
      "loss": 0.0028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.2163848876953125,
      "rewards/margins": 8.244821548461914,
      "rewards/rejected": -12.461207389831543,
      "step": 1657
    },
    {
      "epoch": 2.17,
      "learning_rate": 4.4914927230106495e-05,
      "logits/chosen": -1.8793325424194336,
      "logits/rejected": -2.0541069507598877,
      "logps/chosen": -225.55633544921875,
      "logps/rejected": -350.17144775390625,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.772604942321777,
      "rewards/margins": 7.690614223480225,
      "rewards/rejected": -12.463218688964844,
      "step": 1658
    },
    {
      "epoch": 2.17,
      "learning_rate": 4.490862866447191e-05,
      "logits/chosen": -1.852423906326294,
      "logits/rejected": -1.8437042236328125,
      "logps/chosen": -193.12823486328125,
      "logps/rejected": -297.361572265625,
      "loss": 0.0886,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.9329285621643066,
      "rewards/margins": 7.062554836273193,
      "rewards/rejected": -10.995484352111816,
      "step": 1659
    },
    {
      "epoch": 2.17,
      "learning_rate": 4.4902326642641095e-05,
      "logits/chosen": -2.0002667903900146,
      "logits/rejected": -1.9759914875030518,
      "logps/chosen": -230.4566650390625,
      "logps/rejected": -313.8236389160156,
      "loss": 0.1312,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.2740044593811035,
      "rewards/margins": 7.39484167098999,
      "rewards/rejected": -10.668846130371094,
      "step": 1660
    },
    {
      "epoch": 2.17,
      "learning_rate": 4.489602116570809e-05,
      "logits/chosen": -1.8080790042877197,
      "logits/rejected": -1.86578369140625,
      "logps/chosen": -199.6761016845703,
      "logps/rejected": -272.2088623046875,
      "loss": 0.0626,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.074660301208496,
      "rewards/margins": 6.800763130187988,
      "rewards/rejected": -10.875423431396484,
      "step": 1661
    },
    {
      "epoch": 2.18,
      "learning_rate": 4.488971223476753e-05,
      "logits/chosen": -2.065697431564331,
      "logits/rejected": -2.094249725341797,
      "logps/chosen": -219.26934814453125,
      "logps/rejected": -289.54949951171875,
      "loss": 0.0055,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.626599073410034,
      "rewards/margins": 7.229710102081299,
      "rewards/rejected": -10.85630989074707,
      "step": 1662
    },
    {
      "epoch": 2.18,
      "learning_rate": 4.4883399850914684e-05,
      "logits/chosen": -2.0257272720336914,
      "logits/rejected": -2.0586395263671875,
      "logps/chosen": -210.08489990234375,
      "logps/rejected": -290.2291259765625,
      "loss": 0.1383,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.094170570373535,
      "rewards/margins": 6.843661308288574,
      "rewards/rejected": -10.93783187866211,
      "step": 1663
    },
    {
      "epoch": 2.18,
      "learning_rate": 4.4877084015245394e-05,
      "logits/chosen": -2.095829963684082,
      "logits/rejected": -2.090388059616089,
      "logps/chosen": -216.79452514648438,
      "logps/rejected": -315.7300720214844,
      "loss": 0.0903,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.351548194885254,
      "rewards/margins": 7.777657508850098,
      "rewards/rejected": -12.129206657409668,
      "step": 1664
    },
    {
      "epoch": 2.18,
      "learning_rate": 4.48707647288561e-05,
      "logits/chosen": -1.8059430122375488,
      "logits/rejected": -1.9512474536895752,
      "logps/chosen": -220.30703735351562,
      "logps/rejected": -398.6494445800781,
      "loss": 0.0036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.140538215637207,
      "rewards/margins": 9.172012329101562,
      "rewards/rejected": -13.31255054473877,
      "step": 1665
    },
    {
      "epoch": 2.18,
      "learning_rate": 4.486444199284386e-05,
      "logits/chosen": -1.8825180530548096,
      "logits/rejected": -1.9374803304672241,
      "logps/chosen": -256.9749755859375,
      "logps/rejected": -345.34686279296875,
      "loss": 0.0065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.18184757232666,
      "rewards/margins": 9.528462409973145,
      "rewards/rejected": -13.710309982299805,
      "step": 1666
    },
    {
      "epoch": 2.18,
      "learning_rate": 4.4858115808306314e-05,
      "logits/chosen": -1.770180106163025,
      "logits/rejected": -1.740568995475769,
      "logps/chosen": -235.30775451660156,
      "logps/rejected": -320.1776428222656,
      "loss": 0.0181,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.58617639541626,
      "rewards/margins": 7.5318989753723145,
      "rewards/rejected": -12.118075370788574,
      "step": 1667
    },
    {
      "epoch": 2.18,
      "learning_rate": 4.48517861763417e-05,
      "logits/chosen": -1.7661123275756836,
      "logits/rejected": -1.7018529176712036,
      "logps/chosen": -224.44650268554688,
      "logps/rejected": -286.1048278808594,
      "loss": 0.0454,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.588602066040039,
      "rewards/margins": 7.2582621574401855,
      "rewards/rejected": -10.846864700317383,
      "step": 1668
    },
    {
      "epoch": 2.18,
      "learning_rate": 4.4845453098048876e-05,
      "logits/chosen": -1.9162724018096924,
      "logits/rejected": -1.9782600402832031,
      "logps/chosen": -252.06277465820312,
      "logps/rejected": -341.20379638671875,
      "loss": 0.1005,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.865740776062012,
      "rewards/margins": 7.253775119781494,
      "rewards/rejected": -12.119516372680664,
      "step": 1669
    },
    {
      "epoch": 2.19,
      "learning_rate": 4.483911657452726e-05,
      "logits/chosen": -2.012774705886841,
      "logits/rejected": -2.090327501296997,
      "logps/chosen": -203.31741333007812,
      "logps/rejected": -287.6564025878906,
      "loss": 0.1608,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.810271739959717,
      "rewards/margins": 6.6673126220703125,
      "rewards/rejected": -10.477583885192871,
      "step": 1670
    },
    {
      "epoch": 2.19,
      "learning_rate": 4.483277660687692e-05,
      "logits/chosen": -1.927455186843872,
      "logits/rejected": -1.9408812522888184,
      "logps/chosen": -276.76416015625,
      "logps/rejected": -364.72320556640625,
      "loss": 0.0474,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.065609931945801,
      "rewards/margins": 7.9829230308532715,
      "rewards/rejected": -12.04853343963623,
      "step": 1671
    },
    {
      "epoch": 2.19,
      "learning_rate": 4.482643319619847e-05,
      "logits/chosen": -1.7971769571304321,
      "logits/rejected": -1.834416389465332,
      "logps/chosen": -247.7070770263672,
      "logps/rejected": -356.5272216796875,
      "loss": 0.0897,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.6214561462402344,
      "rewards/margins": 8.43688678741455,
      "rewards/rejected": -12.058342933654785,
      "step": 1672
    },
    {
      "epoch": 2.19,
      "learning_rate": 4.482008634359315e-05,
      "logits/chosen": -1.9927940368652344,
      "logits/rejected": -2.031109094619751,
      "logps/chosen": -237.0690460205078,
      "logps/rejected": -314.8182373046875,
      "loss": 0.0932,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.718445777893066,
      "rewards/margins": 7.051286220550537,
      "rewards/rejected": -11.769731521606445,
      "step": 1673
    },
    {
      "epoch": 2.19,
      "learning_rate": 4.48137360501628e-05,
      "logits/chosen": -1.9173270463943481,
      "logits/rejected": -1.9167189598083496,
      "logps/chosen": -203.08154296875,
      "logps/rejected": -287.092041015625,
      "loss": 0.0885,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.5145535469055176,
      "rewards/margins": 7.844388484954834,
      "rewards/rejected": -11.358942985534668,
      "step": 1674
    },
    {
      "epoch": 2.19,
      "learning_rate": 4.480738231700985e-05,
      "logits/chosen": -2.1176838874816895,
      "logits/rejected": -2.0974013805389404,
      "logps/chosen": -266.84588623046875,
      "logps/rejected": -359.0920104980469,
      "loss": 0.0445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.568906784057617,
      "rewards/margins": 9.128886222839355,
      "rewards/rejected": -13.697793006896973,
      "step": 1675
    },
    {
      "epoch": 2.19,
      "learning_rate": 4.4801025145237306e-05,
      "logits/chosen": -1.902029037475586,
      "logits/rejected": -1.8923699855804443,
      "logps/chosen": -227.2543487548828,
      "logps/rejected": -288.05206298828125,
      "loss": 0.1307,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.330000400543213,
      "rewards/margins": 7.22145938873291,
      "rewards/rejected": -10.551460266113281,
      "step": 1676
    },
    {
      "epoch": 2.19,
      "learning_rate": 4.4794664535948816e-05,
      "logits/chosen": -1.6977206468582153,
      "logits/rejected": -1.8270032405853271,
      "logps/chosen": -223.025390625,
      "logps/rejected": -359.58563232421875,
      "loss": 0.0053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.561017036437988,
      "rewards/margins": 8.420669555664062,
      "rewards/rejected": -12.981685638427734,
      "step": 1677
    },
    {
      "epoch": 2.2,
      "learning_rate": 4.478830049024859e-05,
      "logits/chosen": -2.022806167602539,
      "logits/rejected": -1.9440600872039795,
      "logps/chosen": -224.28826904296875,
      "logps/rejected": -275.18988037109375,
      "loss": 0.0473,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.236040115356445,
      "rewards/margins": 7.757159233093262,
      "rewards/rejected": -11.993199348449707,
      "step": 1678
    },
    {
      "epoch": 2.2,
      "learning_rate": 4.478193300924145e-05,
      "logits/chosen": -2.131181001663208,
      "logits/rejected": -2.179513931274414,
      "logps/chosen": -221.23886108398438,
      "logps/rejected": -285.28485107421875,
      "loss": 0.0884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.4926037788391113,
      "rewards/margins": 7.22347354888916,
      "rewards/rejected": -10.716076850891113,
      "step": 1679
    },
    {
      "epoch": 2.2,
      "learning_rate": 4.4775562094032795e-05,
      "logits/chosen": -1.5273998975753784,
      "logits/rejected": -1.7143009901046753,
      "logps/chosen": -271.4801025390625,
      "logps/rejected": -375.77069091796875,
      "loss": 0.1006,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.011230945587158,
      "rewards/margins": 7.899630069732666,
      "rewards/rejected": -13.910860061645508,
      "step": 1680
    },
    {
      "epoch": 2.2,
      "learning_rate": 4.476918774572865e-05,
      "logits/chosen": -1.828747272491455,
      "logits/rejected": -1.9121427536010742,
      "logps/chosen": -214.4183807373047,
      "logps/rejected": -285.906982421875,
      "loss": 0.0648,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.341349124908447,
      "rewards/margins": 6.82589054107666,
      "rewards/rejected": -12.16723918914795,
      "step": 1681
    },
    {
      "epoch": 2.2,
      "learning_rate": 4.476280996543561e-05,
      "logits/chosen": -1.9479432106018066,
      "logits/rejected": -2.016745090484619,
      "logps/chosen": -272.9812316894531,
      "logps/rejected": -366.86865234375,
      "loss": 0.0759,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.882082462310791,
      "rewards/margins": 8.156986236572266,
      "rewards/rejected": -12.039068222045898,
      "step": 1682
    },
    {
      "epoch": 2.2,
      "learning_rate": 4.47564287542609e-05,
      "logits/chosen": -1.7548673152923584,
      "logits/rejected": -1.747685432434082,
      "logps/chosen": -233.76705932617188,
      "logps/rejected": -317.80145263671875,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.551597595214844,
      "rewards/margins": 8.004530906677246,
      "rewards/rejected": -12.556129455566406,
      "step": 1683
    },
    {
      "epoch": 2.2,
      "learning_rate": 4.4750044113312284e-05,
      "logits/chosen": -1.8895262479782104,
      "logits/rejected": -1.8817856311798096,
      "logps/chosen": -243.14794921875,
      "logps/rejected": -309.8323974609375,
      "loss": 0.0467,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.921259880065918,
      "rewards/margins": 7.604479789733887,
      "rewards/rejected": -12.525739669799805,
      "step": 1684
    },
    {
      "epoch": 2.21,
      "learning_rate": 4.474365604369817e-05,
      "logits/chosen": -1.8540509939193726,
      "logits/rejected": -1.894709587097168,
      "logps/chosen": -225.369384765625,
      "logps/rejected": -333.721923828125,
      "loss": 0.096,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.20390510559082,
      "rewards/margins": 7.378586769104004,
      "rewards/rejected": -12.582491874694824,
      "step": 1685
    },
    {
      "epoch": 2.21,
      "learning_rate": 4.473726454652755e-05,
      "logits/chosen": -1.8880846500396729,
      "logits/rejected": -1.8266561031341553,
      "logps/chosen": -197.33335876464844,
      "logps/rejected": -272.0766296386719,
      "loss": 0.0456,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.894861936569214,
      "rewards/margins": 7.535285472869873,
      "rewards/rejected": -11.430148124694824,
      "step": 1686
    },
    {
      "epoch": 2.21,
      "learning_rate": 4.473086962290999e-05,
      "logits/chosen": -2.0629053115844727,
      "logits/rejected": -2.107692003250122,
      "logps/chosen": -267.00848388671875,
      "logps/rejected": -336.1997985839844,
      "loss": 0.0952,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.5595641136169434,
      "rewards/margins": 8.624744415283203,
      "rewards/rejected": -12.184309005737305,
      "step": 1687
    },
    {
      "epoch": 2.21,
      "learning_rate": 4.4724471273955686e-05,
      "logits/chosen": -2.018510341644287,
      "logits/rejected": -2.0894618034362793,
      "logps/chosen": -231.053466796875,
      "logps/rejected": -336.80023193359375,
      "loss": 0.0048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.892425537109375,
      "rewards/margins": 8.695682525634766,
      "rewards/rejected": -13.58810806274414,
      "step": 1688
    },
    {
      "epoch": 2.21,
      "learning_rate": 4.471806950077541e-05,
      "logits/chosen": -1.6260873079299927,
      "logits/rejected": -1.654736876487732,
      "logps/chosen": -198.38699340820312,
      "logps/rejected": -290.44287109375,
      "loss": 0.0025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.7985918521881104,
      "rewards/margins": 8.221394538879395,
      "rewards/rejected": -12.019987106323242,
      "step": 1689
    },
    {
      "epoch": 2.21,
      "learning_rate": 4.47116643044805e-05,
      "logits/chosen": -1.8681111335754395,
      "logits/rejected": -1.9466384649276733,
      "logps/chosen": -228.25901794433594,
      "logps/rejected": -361.3291320800781,
      "loss": 0.0038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.587683200836182,
      "rewards/margins": 9.266342163085938,
      "rewards/rejected": -13.854026794433594,
      "step": 1690
    },
    {
      "epoch": 2.21,
      "learning_rate": 4.470525568618296e-05,
      "logits/chosen": -1.870570182800293,
      "logits/rejected": -1.9170101881027222,
      "logps/chosen": -257.67340087890625,
      "logps/rejected": -370.72943115234375,
      "loss": 0.0454,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.8660976886749268,
      "rewards/margins": 11.009435653686523,
      "rewards/rejected": -14.875534057617188,
      "step": 1691
    },
    {
      "epoch": 2.21,
      "learning_rate": 4.469884364699531e-05,
      "logits/chosen": -1.875512957572937,
      "logits/rejected": -2.05307674407959,
      "logps/chosen": -236.80178833007812,
      "logps/rejected": -374.8936767578125,
      "loss": 0.0879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.758052825927734,
      "rewards/margins": 9.715606689453125,
      "rewards/rejected": -14.47365951538086,
      "step": 1692
    },
    {
      "epoch": 2.22,
      "learning_rate": 4.469242818803072e-05,
      "logits/chosen": -1.8573062419891357,
      "logits/rejected": -1.7853858470916748,
      "logps/chosen": -311.140625,
      "logps/rejected": -379.6907653808594,
      "loss": 0.0916,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.95181941986084,
      "rewards/margins": 8.339434623718262,
      "rewards/rejected": -12.291254043579102,
      "step": 1693
    },
    {
      "epoch": 2.22,
      "learning_rate": 4.4686009310402925e-05,
      "logits/chosen": -1.7488088607788086,
      "logits/rejected": -1.765557885169983,
      "logps/chosen": -228.75222778320312,
      "logps/rejected": -324.71331787109375,
      "loss": 0.0484,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.402834415435791,
      "rewards/margins": 8.772923469543457,
      "rewards/rejected": -13.175758361816406,
      "step": 1694
    },
    {
      "epoch": 2.22,
      "learning_rate": 4.4679587015226253e-05,
      "logits/chosen": -1.8654567003250122,
      "logits/rejected": -1.9250493049621582,
      "logps/chosen": -239.89849853515625,
      "logps/rejected": -349.3185119628906,
      "loss": 0.0413,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.728438854217529,
      "rewards/margins": 8.32399845123291,
      "rewards/rejected": -14.052436828613281,
      "step": 1695
    },
    {
      "epoch": 2.22,
      "learning_rate": 4.467316130361565e-05,
      "logits/chosen": -1.8583791255950928,
      "logits/rejected": -1.8942469358444214,
      "logps/chosen": -237.06396484375,
      "logps/rejected": -343.1637878417969,
      "loss": 0.0162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.936460018157959,
      "rewards/margins": 8.794307708740234,
      "rewards/rejected": -13.730766296386719,
      "step": 1696
    },
    {
      "epoch": 2.22,
      "learning_rate": 4.466673217668662e-05,
      "logits/chosen": -2.079540252685547,
      "logits/rejected": -2.164118528366089,
      "logps/chosen": -279.9512023925781,
      "logps/rejected": -423.8478698730469,
      "loss": 0.0436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.133212566375732,
      "rewards/margins": 10.697543144226074,
      "rewards/rejected": -15.830756187438965,
      "step": 1697
    },
    {
      "epoch": 2.22,
      "learning_rate": 4.466029963555529e-05,
      "logits/chosen": -1.917163372039795,
      "logits/rejected": -1.8837778568267822,
      "logps/chosen": -233.43521118164062,
      "logps/rejected": -284.9217834472656,
      "loss": 0.0964,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.6208252906799316,
      "rewards/margins": 6.869645595550537,
      "rewards/rejected": -10.490470886230469,
      "step": 1698
    },
    {
      "epoch": 2.22,
      "learning_rate": 4.465386368133837e-05,
      "logits/chosen": -1.918818473815918,
      "logits/rejected": -1.984576940536499,
      "logps/chosen": -248.1490936279297,
      "logps/rejected": -338.5713806152344,
      "loss": 0.0441,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.999472141265869,
      "rewards/margins": 7.363437652587891,
      "rewards/rejected": -13.362909317016602,
      "step": 1699
    },
    {
      "epoch": 2.22,
      "learning_rate": 4.464742431515315e-05,
      "logits/chosen": -1.8976091146469116,
      "logits/rejected": -1.9875017404556274,
      "logps/chosen": -208.65817260742188,
      "logps/rejected": -305.20428466796875,
      "loss": 0.1059,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.453165531158447,
      "rewards/margins": 9.020037651062012,
      "rewards/rejected": -13.473203659057617,
      "step": 1700
    },
    {
      "epoch": 2.23,
      "learning_rate": 4.4640981538117534e-05,
      "logits/chosen": -1.6591687202453613,
      "logits/rejected": -1.6958729028701782,
      "logps/chosen": -226.06491088867188,
      "logps/rejected": -328.7708740234375,
      "loss": 0.048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.8626770973205566,
      "rewards/margins": 9.48762321472168,
      "rewards/rejected": -13.350301742553711,
      "step": 1701
    },
    {
      "epoch": 2.23,
      "learning_rate": 4.4634535351349995e-05,
      "logits/chosen": -1.8268656730651855,
      "logits/rejected": -1.9317461252212524,
      "logps/chosen": -192.97369384765625,
      "logps/rejected": -290.1656494140625,
      "loss": 0.0599,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.650546550750732,
      "rewards/margins": 7.853985786437988,
      "rewards/rejected": -12.504531860351562,
      "step": 1702
    },
    {
      "epoch": 2.23,
      "learning_rate": 4.462808575596962e-05,
      "logits/chosen": -1.9132554531097412,
      "logits/rejected": -1.965674877166748,
      "logps/chosen": -218.21656799316406,
      "logps/rejected": -364.0094909667969,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.798173427581787,
      "rewards/margins": 9.15748119354248,
      "rewards/rejected": -13.955655097961426,
      "step": 1703
    },
    {
      "epoch": 2.23,
      "learning_rate": 4.4621632753096066e-05,
      "logits/chosen": -1.9854216575622559,
      "logits/rejected": -2.0571200847625732,
      "logps/chosen": -252.40103149414062,
      "logps/rejected": -309.4356689453125,
      "loss": 0.071,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.203364372253418,
      "rewards/margins": 6.705964088439941,
      "rewards/rejected": -11.90932846069336,
      "step": 1704
    },
    {
      "epoch": 2.23,
      "learning_rate": 4.461517634384961e-05,
      "logits/chosen": -1.6782927513122559,
      "logits/rejected": -1.6738965511322021,
      "logps/chosen": -216.7491912841797,
      "logps/rejected": -303.1639709472656,
      "loss": 0.1341,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.4354166984558105,
      "rewards/margins": 7.658374786376953,
      "rewards/rejected": -11.093792915344238,
      "step": 1705
    },
    {
      "epoch": 2.23,
      "learning_rate": 4.460871652935109e-05,
      "logits/chosen": -1.87294602394104,
      "logits/rejected": -1.9136104583740234,
      "logps/chosen": -267.9233093261719,
      "logps/rejected": -348.4906921386719,
      "loss": 0.1321,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.544414043426514,
      "rewards/margins": 7.048198699951172,
      "rewards/rejected": -11.59261417388916,
      "step": 1706
    },
    {
      "epoch": 2.23,
      "learning_rate": 4.4602253310721964e-05,
      "logits/chosen": -1.7989273071289062,
      "logits/rejected": -1.8650834560394287,
      "logps/chosen": -252.19677734375,
      "logps/rejected": -340.9962158203125,
      "loss": 0.0045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.7554030418396,
      "rewards/margins": 8.529303550720215,
      "rewards/rejected": -13.284706115722656,
      "step": 1707
    },
    {
      "epoch": 2.24,
      "learning_rate": 4.4595786689084246e-05,
      "logits/chosen": -2.063161611557007,
      "logits/rejected": -2.0948920249938965,
      "logps/chosen": -233.23056030273438,
      "logps/rejected": -325.5529479980469,
      "loss": 0.0056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.490490674972534,
      "rewards/margins": 10.117737770080566,
      "rewards/rejected": -13.608229637145996,
      "step": 1708
    },
    {
      "epoch": 2.24,
      "learning_rate": 4.4589316665560564e-05,
      "logits/chosen": -1.9486212730407715,
      "logits/rejected": -1.88687264919281,
      "logps/chosen": -288.6056213378906,
      "logps/rejected": -419.3163757324219,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.242236137390137,
      "rewards/margins": 11.447809219360352,
      "rewards/rejected": -15.690044403076172,
      "step": 1709
    },
    {
      "epoch": 2.24,
      "learning_rate": 4.458284324127415e-05,
      "logits/chosen": -1.4820352792739868,
      "logits/rejected": -1.5303387641906738,
      "logps/chosen": -222.28982543945312,
      "logps/rejected": -320.25238037109375,
      "loss": 0.0892,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.2132415771484375,
      "rewards/margins": 8.695575714111328,
      "rewards/rejected": -12.908817291259766,
      "step": 1710
    },
    {
      "epoch": 2.24,
      "learning_rate": 4.457636641734879e-05,
      "logits/chosen": -1.919441819190979,
      "logits/rejected": -1.820131540298462,
      "logps/chosen": -202.3666534423828,
      "logps/rejected": -269.91851806640625,
      "loss": 0.0526,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.528416633605957,
      "rewards/margins": 9.030194282531738,
      "rewards/rejected": -12.558610916137695,
      "step": 1711
    },
    {
      "epoch": 2.24,
      "learning_rate": 4.456988619490889e-05,
      "logits/chosen": -1.9204466342926025,
      "logits/rejected": -1.8682175874710083,
      "logps/chosen": -261.3448181152344,
      "logps/rejected": -361.8304443359375,
      "loss": 0.0871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.659709930419922,
      "rewards/margins": 10.16419506072998,
      "rewards/rejected": -14.823904991149902,
      "step": 1712
    },
    {
      "epoch": 2.24,
      "learning_rate": 4.456340257507944e-05,
      "logits/chosen": -1.9544986486434937,
      "logits/rejected": -2.006072759628296,
      "logps/chosen": -173.63169860839844,
      "logps/rejected": -272.7809143066406,
      "loss": 0.0544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.102273941040039,
      "rewards/margins": 7.849088668823242,
      "rewards/rejected": -10.951361656188965,
      "step": 1713
    },
    {
      "epoch": 2.24,
      "learning_rate": 4.4556915558986e-05,
      "logits/chosen": -2.0595617294311523,
      "logits/rejected": -2.0527684688568115,
      "logps/chosen": -249.13677978515625,
      "logps/rejected": -318.8485107421875,
      "loss": 0.0058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.425116062164307,
      "rewards/margins": 9.156496047973633,
      "rewards/rejected": -13.581611633300781,
      "step": 1714
    },
    {
      "epoch": 2.24,
      "learning_rate": 4.455042514775475e-05,
      "logits/chosen": -2.034576416015625,
      "logits/rejected": -2.082951784133911,
      "logps/chosen": -238.15553283691406,
      "logps/rejected": -313.01251220703125,
      "loss": 0.0878,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.427706241607666,
      "rewards/margins": 7.517872333526611,
      "rewards/rejected": -11.945578575134277,
      "step": 1715
    },
    {
      "epoch": 2.25,
      "learning_rate": 4.454393134251243e-05,
      "logits/chosen": -1.9722774028778076,
      "logits/rejected": -2.0078694820404053,
      "logps/chosen": -214.91195678710938,
      "logps/rejected": -305.921630859375,
      "loss": 0.0893,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.9531149864196777,
      "rewards/margins": 8.302417755126953,
      "rewards/rejected": -12.255531311035156,
      "step": 1716
    },
    {
      "epoch": 2.25,
      "learning_rate": 4.4537434144386406e-05,
      "logits/chosen": -2.0352540016174316,
      "logits/rejected": -2.0985212326049805,
      "logps/chosen": -213.61550903320312,
      "logps/rejected": -310.8976745605469,
      "loss": 0.0445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.662770748138428,
      "rewards/margins": 8.683988571166992,
      "rewards/rejected": -13.346759796142578,
      "step": 1717
    },
    {
      "epoch": 2.25,
      "learning_rate": 4.4530933554504584e-05,
      "logits/chosen": -1.7264947891235352,
      "logits/rejected": -1.7682827711105347,
      "logps/chosen": -173.7081298828125,
      "logps/rejected": -313.7381286621094,
      "loss": 0.0953,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.215333461761475,
      "rewards/margins": 8.603334426879883,
      "rewards/rejected": -12.8186674118042,
      "step": 1718
    },
    {
      "epoch": 2.25,
      "learning_rate": 4.45244295739955e-05,
      "logits/chosen": -1.9386180639266968,
      "logits/rejected": -2.0219626426696777,
      "logps/chosen": -310.0873107910156,
      "logps/rejected": -405.73297119140625,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.907812595367432,
      "rewards/margins": 11.40237045288086,
      "rewards/rejected": -16.310182571411133,
      "step": 1719
    },
    {
      "epoch": 2.25,
      "learning_rate": 4.451792220398826e-05,
      "logits/chosen": -1.8895277976989746,
      "logits/rejected": -1.7938296794891357,
      "logps/chosen": -282.5625305175781,
      "logps/rejected": -364.5167236328125,
      "loss": 0.0722,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.161428928375244,
      "rewards/margins": 8.818914413452148,
      "rewards/rejected": -13.98034381866455,
      "step": 1720
    },
    {
      "epoch": 2.25,
      "learning_rate": 4.4511411445612574e-05,
      "logits/chosen": -2.0378122329711914,
      "logits/rejected": -2.16402006149292,
      "logps/chosen": -252.35549926757812,
      "logps/rejected": -356.19903564453125,
      "loss": 0.1538,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.731842994689941,
      "rewards/margins": 6.997723579406738,
      "rewards/rejected": -12.729567527770996,
      "step": 1721
    },
    {
      "epoch": 2.25,
      "learning_rate": 4.450489729999871e-05,
      "logits/chosen": -1.8221724033355713,
      "logits/rejected": -1.8675659894943237,
      "logps/chosen": -201.3778076171875,
      "logps/rejected": -327.18841552734375,
      "loss": 0.0127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0520832538604736,
      "rewards/margins": 10.92273998260498,
      "rewards/rejected": -13.974822044372559,
      "step": 1722
    },
    {
      "epoch": 2.25,
      "learning_rate": 4.4498379768277555e-05,
      "logits/chosen": -1.6914008855819702,
      "logits/rejected": -1.7110141515731812,
      "logps/chosen": -222.4412841796875,
      "logps/rejected": -326.3401184082031,
      "loss": 0.0122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.712990760803223,
      "rewards/margins": 7.869570732116699,
      "rewards/rejected": -12.582560539245605,
      "step": 1723
    },
    {
      "epoch": 2.26,
      "learning_rate": 4.449185885158056e-05,
      "logits/chosen": -1.650061845779419,
      "logits/rejected": -1.771134614944458,
      "logps/chosen": -200.29811096191406,
      "logps/rejected": -301.3843688964844,
      "loss": 0.0467,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.501410961151123,
      "rewards/margins": 8.028314590454102,
      "rewards/rejected": -12.529725074768066,
      "step": 1724
    },
    {
      "epoch": 2.26,
      "learning_rate": 4.448533455103979e-05,
      "logits/chosen": -1.8757323026657104,
      "logits/rejected": -1.9865539073944092,
      "logps/chosen": -233.38584899902344,
      "logps/rejected": -327.2201232910156,
      "loss": 0.0065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.171833038330078,
      "rewards/margins": 7.656855583190918,
      "rewards/rejected": -12.82868766784668,
      "step": 1725
    },
    {
      "epoch": 2.26,
      "learning_rate": 4.447880686778788e-05,
      "logits/chosen": -1.9543880224227905,
      "logits/rejected": -1.994931697845459,
      "logps/chosen": -211.9916229248047,
      "logps/rejected": -283.35003662109375,
      "loss": 0.0667,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.0559065341949463,
      "rewards/margins": 8.326825141906738,
      "rewards/rejected": -11.382730484008789,
      "step": 1726
    },
    {
      "epoch": 2.26,
      "learning_rate": 4.447227580295805e-05,
      "logits/chosen": -2.1561076641082764,
      "logits/rejected": -2.169799327850342,
      "logps/chosen": -227.1396484375,
      "logps/rejected": -322.3929443359375,
      "loss": 0.1708,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.8819479942321777,
      "rewards/margins": 8.353205680847168,
      "rewards/rejected": -12.235153198242188,
      "step": 1727
    },
    {
      "epoch": 2.26,
      "learning_rate": 4.446574135768411e-05,
      "logits/chosen": -1.7280957698822021,
      "logits/rejected": -1.8162102699279785,
      "logps/chosen": -219.45413208007812,
      "logps/rejected": -305.6361083984375,
      "loss": 0.0064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.295023441314697,
      "rewards/margins": 9.003251075744629,
      "rewards/rejected": -13.298274993896484,
      "step": 1728
    },
    {
      "epoch": 2.26,
      "learning_rate": 4.445920353310046e-05,
      "logits/chosen": -2.0517849922180176,
      "logits/rejected": -2.169731378555298,
      "logps/chosen": -244.9127655029297,
      "logps/rejected": -339.7769470214844,
      "loss": 0.0464,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.833156585693359,
      "rewards/margins": 8.40161418914795,
      "rewards/rejected": -13.234771728515625,
      "step": 1729
    },
    {
      "epoch": 2.26,
      "learning_rate": 4.445266233034209e-05,
      "logits/chosen": -1.9063819646835327,
      "logits/rejected": -1.9939993619918823,
      "logps/chosen": -210.46400451660156,
      "logps/rejected": -321.38134765625,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.046882629394531,
      "rewards/margins": 9.02386474609375,
      "rewards/rejected": -13.070748329162598,
      "step": 1730
    },
    {
      "epoch": 2.27,
      "learning_rate": 4.444611775054457e-05,
      "logits/chosen": -2.016284942626953,
      "logits/rejected": -2.0195984840393066,
      "logps/chosen": -220.59742736816406,
      "logps/rejected": -260.17816162109375,
      "loss": 0.0771,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.4410529136657715,
      "rewards/margins": 6.340338706970215,
      "rewards/rejected": -9.781391143798828,
      "step": 1731
    },
    {
      "epoch": 2.27,
      "learning_rate": 4.443956979484406e-05,
      "logits/chosen": -1.9992839097976685,
      "logits/rejected": -1.943981647491455,
      "logps/chosen": -236.16993713378906,
      "logps/rejected": -313.77069091796875,
      "loss": 0.0151,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.058987617492676,
      "rewards/margins": 8.646703720092773,
      "rewards/rejected": -12.70569133758545,
      "step": 1732
    },
    {
      "epoch": 2.27,
      "learning_rate": 4.44330184643773e-05,
      "logits/chosen": -1.8650450706481934,
      "logits/rejected": -1.7884596586227417,
      "logps/chosen": -224.2449188232422,
      "logps/rejected": -271.3692321777344,
      "loss": 0.1366,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.8722286224365234,
      "rewards/margins": 6.974151134490967,
      "rewards/rejected": -10.846380233764648,
      "step": 1733
    },
    {
      "epoch": 2.27,
      "learning_rate": 4.442646376028162e-05,
      "logits/chosen": -1.7677942514419556,
      "logits/rejected": -1.826995611190796,
      "logps/chosen": -224.70472717285156,
      "logps/rejected": -334.978271484375,
      "loss": 0.1076,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.9479570388793945,
      "rewards/margins": 8.14836311340332,
      "rewards/rejected": -13.096320152282715,
      "step": 1734
    },
    {
      "epoch": 2.27,
      "learning_rate": 4.441990568369494e-05,
      "logits/chosen": -1.9324272871017456,
      "logits/rejected": -2.0757665634155273,
      "logps/chosen": -217.09535217285156,
      "logps/rejected": -313.4742736816406,
      "loss": 0.0459,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.570109844207764,
      "rewards/margins": 8.275823593139648,
      "rewards/rejected": -12.84593391418457,
      "step": 1735
    },
    {
      "epoch": 2.27,
      "learning_rate": 4.441334423575576e-05,
      "logits/chosen": -1.9709575176239014,
      "logits/rejected": -1.8980768918991089,
      "logps/chosen": -233.23419189453125,
      "logps/rejected": -291.5000305175781,
      "loss": 0.0918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.155413627624512,
      "rewards/margins": 5.9400105476379395,
      "rewards/rejected": -10.09542465209961,
      "step": 1736
    },
    {
      "epoch": 2.27,
      "learning_rate": 4.440677941760316e-05,
      "logits/chosen": -1.8670390844345093,
      "logits/rejected": -1.864248275756836,
      "logps/chosen": -177.4266815185547,
      "logps/rejected": -256.11529541015625,
      "loss": 0.0491,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.6511425971984863,
      "rewards/margins": 6.640891075134277,
      "rewards/rejected": -10.292034149169922,
      "step": 1737
    },
    {
      "epoch": 2.27,
      "learning_rate": 4.4400211230376826e-05,
      "logits/chosen": -2.05721378326416,
      "logits/rejected": -2.158565044403076,
      "logps/chosen": -260.0497741699219,
      "logps/rejected": -376.5098571777344,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.747610569000244,
      "rewards/margins": 9.646328926086426,
      "rewards/rejected": -14.393939971923828,
      "step": 1738
    },
    {
      "epoch": 2.28,
      "learning_rate": 4.4393639675217004e-05,
      "logits/chosen": -1.7898157835006714,
      "logits/rejected": -1.9009355306625366,
      "logps/chosen": -203.9080352783203,
      "logps/rejected": -296.2388610839844,
      "loss": 0.095,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.05819034576416,
      "rewards/margins": 6.971330165863037,
      "rewards/rejected": -11.029520034790039,
      "step": 1739
    },
    {
      "epoch": 2.28,
      "learning_rate": 4.438706475326453e-05,
      "logits/chosen": -1.7593650817871094,
      "logits/rejected": -1.8225117921829224,
      "logps/chosen": -205.5297088623047,
      "logps/rejected": -274.93328857421875,
      "loss": 0.0872,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.9774317741394043,
      "rewards/margins": 7.377807140350342,
      "rewards/rejected": -11.355239868164062,
      "step": 1740
    },
    {
      "epoch": 2.28,
      "learning_rate": 4.438048646566084e-05,
      "logits/chosen": -1.8858990669250488,
      "logits/rejected": -1.9695152044296265,
      "logps/chosen": -215.17010498046875,
      "logps/rejected": -334.5107727050781,
      "loss": 0.0057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.608569622039795,
      "rewards/margins": 8.680890083312988,
      "rewards/rejected": -12.289458274841309,
      "step": 1741
    },
    {
      "epoch": 2.28,
      "learning_rate": 4.437390481354795e-05,
      "logits/chosen": -2.0719106197357178,
      "logits/rejected": -2.1239736080169678,
      "logps/chosen": -251.9342041015625,
      "logps/rejected": -374.99908447265625,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.334467887878418,
      "rewards/margins": 9.548616409301758,
      "rewards/rejected": -13.883084297180176,
      "step": 1742
    },
    {
      "epoch": 2.28,
      "learning_rate": 4.436731979806844e-05,
      "logits/chosen": -1.8284627199172974,
      "logits/rejected": -1.918864369392395,
      "logps/chosen": -195.5998077392578,
      "logps/rejected": -328.7000732421875,
      "loss": 0.0021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.969945192337036,
      "rewards/margins": 9.359249114990234,
      "rewards/rejected": -13.329195022583008,
      "step": 1743
    },
    {
      "epoch": 2.28,
      "learning_rate": 4.4360731420365486e-05,
      "logits/chosen": -1.6309916973114014,
      "logits/rejected": -1.718712329864502,
      "logps/chosen": -202.272705078125,
      "logps/rejected": -307.3332214355469,
      "loss": 0.0657,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.2723641395568848,
      "rewards/margins": 8.126914024353027,
      "rewards/rejected": -11.39927864074707,
      "step": 1744
    },
    {
      "epoch": 2.28,
      "learning_rate": 4.4354139681582865e-05,
      "logits/chosen": -1.9175488948822021,
      "logits/rejected": -1.9487149715423584,
      "logps/chosen": -284.1695556640625,
      "logps/rejected": -303.47442626953125,
      "loss": 0.0522,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.2527360916137695,
      "rewards/margins": 7.352540016174316,
      "rewards/rejected": -11.605276107788086,
      "step": 1745
    },
    {
      "epoch": 2.28,
      "learning_rate": 4.434754458286491e-05,
      "logits/chosen": -2.0459342002868652,
      "logits/rejected": -2.0865938663482666,
      "logps/chosen": -235.94528198242188,
      "logps/rejected": -334.06109619140625,
      "loss": 0.0877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.6131935119628906,
      "rewards/margins": 8.025009155273438,
      "rewards/rejected": -11.638202667236328,
      "step": 1746
    },
    {
      "epoch": 2.29,
      "learning_rate": 4.434094612535655e-05,
      "logits/chosen": -2.1044609546661377,
      "logits/rejected": -2.0698766708374023,
      "logps/chosen": -233.37014770507812,
      "logps/rejected": -340.31756591796875,
      "loss": 0.0065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.785109043121338,
      "rewards/margins": 9.152952194213867,
      "rewards/rejected": -13.938060760498047,
      "step": 1747
    },
    {
      "epoch": 2.29,
      "learning_rate": 4.433434431020331e-05,
      "logits/chosen": -1.820961594581604,
      "logits/rejected": -1.9584262371063232,
      "logps/chosen": -175.23321533203125,
      "logps/rejected": -304.24334716796875,
      "loss": 0.1015,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.8073019981384277,
      "rewards/margins": 7.9026360511779785,
      "rewards/rejected": -10.709938049316406,
      "step": 1748
    },
    {
      "epoch": 2.29,
      "learning_rate": 4.432773913855127e-05,
      "logits/chosen": -1.9037257432937622,
      "logits/rejected": -1.7631274461746216,
      "logps/chosen": -224.0203857421875,
      "logps/rejected": -311.80560302734375,
      "loss": 0.0099,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.7373249530792236,
      "rewards/margins": 8.073929786682129,
      "rewards/rejected": -11.811254501342773,
      "step": 1749
    },
    {
      "epoch": 2.29,
      "learning_rate": 4.4321130611547116e-05,
      "logits/chosen": -1.9026196002960205,
      "logits/rejected": -1.9815964698791504,
      "logps/chosen": -212.11672973632812,
      "logps/rejected": -305.43682861328125,
      "loss": 0.166,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.651831865310669,
      "rewards/margins": 6.873808860778809,
      "rewards/rejected": -10.525641441345215,
      "step": 1750
    },
    {
      "epoch": 2.29,
      "learning_rate": 4.431451873033809e-05,
      "logits/chosen": -1.6320197582244873,
      "logits/rejected": -1.6567957401275635,
      "logps/chosen": -198.2225799560547,
      "logps/rejected": -279.5611877441406,
      "loss": 0.1458,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.1111578941345215,
      "rewards/margins": 5.316527366638184,
      "rewards/rejected": -9.427684783935547,
      "step": 1751
    },
    {
      "epoch": 2.29,
      "learning_rate": 4.4307903496072053e-05,
      "logits/chosen": -1.90899658203125,
      "logits/rejected": -1.9217326641082764,
      "logps/chosen": -170.5017547607422,
      "logps/rejected": -242.19381713867188,
      "loss": 0.1412,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.4766669273376465,
      "rewards/margins": 6.857580184936523,
      "rewards/rejected": -10.334247589111328,
      "step": 1752
    },
    {
      "epoch": 2.29,
      "learning_rate": 4.430128490989742e-05,
      "logits/chosen": -2.097954273223877,
      "logits/rejected": -2.082620859146118,
      "logps/chosen": -204.95272827148438,
      "logps/rejected": -305.71075439453125,
      "loss": 0.0723,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.122200012207031,
      "rewards/margins": 7.713401794433594,
      "rewards/rejected": -12.835600852966309,
      "step": 1753
    },
    {
      "epoch": 2.3,
      "learning_rate": 4.429466297296321e-05,
      "logits/chosen": -1.9181946516036987,
      "logits/rejected": -2.054036855697632,
      "logps/chosen": -254.34112548828125,
      "logps/rejected": -373.45904541015625,
      "loss": 0.0446,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.675026893615723,
      "rewards/margins": 8.775556564331055,
      "rewards/rejected": -13.450584411621094,
      "step": 1754
    },
    {
      "epoch": 2.3,
      "learning_rate": 4.4288037686418995e-05,
      "logits/chosen": -1.8053514957427979,
      "logits/rejected": -1.8212918043136597,
      "logps/chosen": -294.88299560546875,
      "logps/rejected": -375.58001708984375,
      "loss": 0.051,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.989837169647217,
      "rewards/margins": 7.898305892944336,
      "rewards/rejected": -12.888142585754395,
      "step": 1755
    },
    {
      "epoch": 2.3,
      "learning_rate": 4.4281409051414944e-05,
      "logits/chosen": -1.6933479309082031,
      "logits/rejected": -1.8062795400619507,
      "logps/chosen": -231.76593017578125,
      "logps/rejected": -350.6939392089844,
      "loss": 0.0874,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.724734306335449,
      "rewards/margins": 8.08170223236084,
      "rewards/rejected": -12.806437492370605,
      "step": 1756
    },
    {
      "epoch": 2.3,
      "learning_rate": 4.427477706910182e-05,
      "logits/chosen": -1.7838891744613647,
      "logits/rejected": -1.9130263328552246,
      "logps/chosen": -270.8051452636719,
      "logps/rejected": -399.8632507324219,
      "loss": 0.0112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.593284606933594,
      "rewards/margins": 9.894659042358398,
      "rewards/rejected": -15.487942695617676,
      "step": 1757
    },
    {
      "epoch": 2.3,
      "learning_rate": 4.426814174063094e-05,
      "logits/chosen": -1.8285061120986938,
      "logits/rejected": -1.9469319581985474,
      "logps/chosen": -194.19351196289062,
      "logps/rejected": -354.72576904296875,
      "loss": 0.0081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.976185321807861,
      "rewards/margins": 7.8145551681518555,
      "rewards/rejected": -12.790740013122559,
      "step": 1758
    },
    {
      "epoch": 2.3,
      "learning_rate": 4.4261503067154234e-05,
      "logits/chosen": -1.8068652153015137,
      "logits/rejected": -1.9476265907287598,
      "logps/chosen": -279.4858703613281,
      "logps/rejected": -366.8956298828125,
      "loss": 0.0441,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.46669864654541,
      "rewards/margins": 9.39613151550293,
      "rewards/rejected": -14.86283016204834,
      "step": 1759
    },
    {
      "epoch": 2.3,
      "learning_rate": 4.425486104982418e-05,
      "logits/chosen": -1.9092851877212524,
      "logits/rejected": -1.862753987312317,
      "logps/chosen": -266.92572021484375,
      "logps/rejected": -353.6373291015625,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.8829591274261475,
      "rewards/margins": 10.231354713439941,
      "rewards/rejected": -14.114313125610352,
      "step": 1760
    },
    {
      "epoch": 2.3,
      "learning_rate": 4.424821568979386e-05,
      "logits/chosen": -1.883470058441162,
      "logits/rejected": -1.826371669769287,
      "logps/chosen": -256.5468444824219,
      "logps/rejected": -334.00390625,
      "loss": 0.0872,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.85820198059082,
      "rewards/margins": 7.960251331329346,
      "rewards/rejected": -12.818452835083008,
      "step": 1761
    },
    {
      "epoch": 2.31,
      "learning_rate": 4.424156698821693e-05,
      "logits/chosen": -1.6825509071350098,
      "logits/rejected": -1.700924038887024,
      "logps/chosen": -253.4066925048828,
      "logps/rejected": -345.5010070800781,
      "loss": 0.162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.353029251098633,
      "rewards/margins": 8.234490394592285,
      "rewards/rejected": -13.587520599365234,
      "step": 1762
    },
    {
      "epoch": 2.31,
      "learning_rate": 4.423491494624761e-05,
      "logits/chosen": -2.003420829772949,
      "logits/rejected": -2.0149574279785156,
      "logps/chosen": -255.87901306152344,
      "logps/rejected": -313.81005859375,
      "loss": 0.0876,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.805246829986572,
      "rewards/margins": 7.625028610229492,
      "rewards/rejected": -12.430275917053223,
      "step": 1763
    },
    {
      "epoch": 2.31,
      "learning_rate": 4.422825956504072e-05,
      "logits/chosen": -1.9779905080795288,
      "logits/rejected": -1.8467421531677246,
      "logps/chosen": -302.4613037109375,
      "logps/rejected": -364.8889465332031,
      "loss": 0.0454,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.031822681427002,
      "rewards/margins": 8.41565990447998,
      "rewards/rejected": -12.447482109069824,
      "step": 1764
    },
    {
      "epoch": 2.31,
      "learning_rate": 4.422160084575167e-05,
      "logits/chosen": -1.8664950132369995,
      "logits/rejected": -1.8366044759750366,
      "logps/chosen": -229.32427978515625,
      "logps/rejected": -269.58538818359375,
      "loss": 0.09,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.961425304412842,
      "rewards/margins": 6.573773384094238,
      "rewards/rejected": -11.535199165344238,
      "step": 1765
    },
    {
      "epoch": 2.31,
      "learning_rate": 4.421493878953641e-05,
      "logits/chosen": -1.8571369647979736,
      "logits/rejected": -1.9223442077636719,
      "logps/chosen": -249.59237670898438,
      "logps/rejected": -351.52716064453125,
      "loss": 0.0433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.7298736572265625,
      "rewards/margins": 8.822587013244629,
      "rewards/rejected": -14.552459716796875,
      "step": 1766
    },
    {
      "epoch": 2.31,
      "learning_rate": 4.42082733975515e-05,
      "logits/chosen": -1.4088032245635986,
      "logits/rejected": -1.5760774612426758,
      "logps/chosen": -260.4433898925781,
      "logps/rejected": -356.95819091796875,
      "loss": 0.0039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.6229753494262695,
      "rewards/margins": 8.967402458190918,
      "rewards/rejected": -14.590376853942871,
      "step": 1767
    },
    {
      "epoch": 2.31,
      "learning_rate": 4.420160467095408e-05,
      "logits/chosen": -2.181647539138794,
      "logits/rejected": -2.090801954269409,
      "logps/chosen": -221.33909606933594,
      "logps/rejected": -306.36138916015625,
      "loss": 0.0528,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.981787919998169,
      "rewards/margins": 9.448159217834473,
      "rewards/rejected": -13.429945945739746,
      "step": 1768
    },
    {
      "epoch": 2.32,
      "learning_rate": 4.419493261090186e-05,
      "logits/chosen": -1.6997369527816772,
      "logits/rejected": -1.7968833446502686,
      "logps/chosen": -195.84988403320312,
      "logps/rejected": -321.94659423828125,
      "loss": 0.0473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.023075103759766,
      "rewards/margins": 8.610339164733887,
      "rewards/rejected": -13.633415222167969,
      "step": 1769
    },
    {
      "epoch": 2.32,
      "learning_rate": 4.418825721855311e-05,
      "logits/chosen": -2.084487199783325,
      "logits/rejected": -2.1054601669311523,
      "logps/chosen": -272.1376037597656,
      "logps/rejected": -358.4344482421875,
      "loss": 0.0478,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.109389781951904,
      "rewards/margins": 7.3886823654174805,
      "rewards/rejected": -12.498072624206543,
      "step": 1770
    },
    {
      "epoch": 2.32,
      "learning_rate": 4.418157849506672e-05,
      "logits/chosen": -1.8278807401657104,
      "logits/rejected": -1.8396012783050537,
      "logps/chosen": -218.77023315429688,
      "logps/rejected": -292.850341796875,
      "loss": 0.142,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.907233715057373,
      "rewards/margins": 7.580648422241211,
      "rewards/rejected": -12.487882614135742,
      "step": 1771
    },
    {
      "epoch": 2.32,
      "learning_rate": 4.417489644160213e-05,
      "logits/chosen": -2.0319652557373047,
      "logits/rejected": -2.0633068084716797,
      "logps/chosen": -260.70452880859375,
      "logps/rejected": -345.8565368652344,
      "loss": 0.0033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.086930274963379,
      "rewards/margins": 9.484430313110352,
      "rewards/rejected": -14.571361541748047,
      "step": 1772
    },
    {
      "epoch": 2.32,
      "learning_rate": 4.4168211059319356e-05,
      "logits/chosen": -1.8316105604171753,
      "logits/rejected": -1.7764731645584106,
      "logps/chosen": -230.56332397460938,
      "logps/rejected": -325.6812744140625,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.026920318603516,
      "rewards/margins": 8.883038520812988,
      "rewards/rejected": -13.909958839416504,
      "step": 1773
    },
    {
      "epoch": 2.32,
      "learning_rate": 4.4161522349379e-05,
      "logits/chosen": -1.7927948236465454,
      "logits/rejected": -1.8635965585708618,
      "logps/chosen": -197.43812561035156,
      "logps/rejected": -288.6591491699219,
      "loss": 0.0454,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.8581838607788086,
      "rewards/margins": 8.61027717590332,
      "rewards/rejected": -12.468461036682129,
      "step": 1774
    },
    {
      "epoch": 2.32,
      "learning_rate": 4.415483031294225e-05,
      "logits/chosen": -1.9382948875427246,
      "logits/rejected": -1.9620851278305054,
      "logps/chosen": -232.10604858398438,
      "logps/rejected": -324.19183349609375,
      "loss": 0.0445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.678308963775635,
      "rewards/margins": 9.27111530303955,
      "rewards/rejected": -13.949423789978027,
      "step": 1775
    },
    {
      "epoch": 2.32,
      "learning_rate": 4.414813495117087e-05,
      "logits/chosen": -1.6935954093933105,
      "logits/rejected": -1.6843981742858887,
      "logps/chosen": -298.6250305175781,
      "logps/rejected": -375.5038146972656,
      "loss": 0.0448,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.873818397521973,
      "rewards/margins": 8.789769172668457,
      "rewards/rejected": -14.66358757019043,
      "step": 1776
    },
    {
      "epoch": 2.33,
      "learning_rate": 4.4141436265227174e-05,
      "logits/chosen": -1.7664049863815308,
      "logits/rejected": -1.801297903060913,
      "logps/chosen": -217.70216369628906,
      "logps/rejected": -327.42449951171875,
      "loss": 0.1323,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.481310844421387,
      "rewards/margins": 7.413915634155273,
      "rewards/rejected": -12.89522647857666,
      "step": 1777
    },
    {
      "epoch": 2.33,
      "learning_rate": 4.413473425627408e-05,
      "logits/chosen": -1.9183545112609863,
      "logits/rejected": -1.9855916500091553,
      "logps/chosen": -220.51705932617188,
      "logps/rejected": -372.94830322265625,
      "loss": 0.0546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.821767807006836,
      "rewards/margins": 10.212467193603516,
      "rewards/rejected": -15.034236907958984,
      "step": 1778
    },
    {
      "epoch": 2.33,
      "learning_rate": 4.4128028925475095e-05,
      "logits/chosen": -2.1654553413391113,
      "logits/rejected": -2.2233729362487793,
      "logps/chosen": -315.9672546386719,
      "logps/rejected": -419.172607421875,
      "loss": 0.0869,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.519159317016602,
      "rewards/margins": 9.854990005493164,
      "rewards/rejected": -15.374149322509766,
      "step": 1779
    },
    {
      "epoch": 2.33,
      "learning_rate": 4.412132027399426e-05,
      "logits/chosen": -2.056398630142212,
      "logits/rejected": -2.127978801727295,
      "logps/chosen": -201.8386993408203,
      "logps/rejected": -338.69744873046875,
      "loss": 0.0939,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.55429744720459,
      "rewards/margins": 8.227093696594238,
      "rewards/rejected": -12.781391143798828,
      "step": 1780
    },
    {
      "epoch": 2.33,
      "learning_rate": 4.411460830299622e-05,
      "logits/chosen": -1.7686713933944702,
      "logits/rejected": -1.9483716487884521,
      "logps/chosen": -258.9558410644531,
      "logps/rejected": -341.24285888671875,
      "loss": 0.1308,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.698920726776123,
      "rewards/margins": 7.26412296295166,
      "rewards/rejected": -13.963045120239258,
      "step": 1781
    },
    {
      "epoch": 2.33,
      "learning_rate": 4.410789301364621e-05,
      "logits/chosen": -2.030712604522705,
      "logits/rejected": -2.0665645599365234,
      "logps/chosen": -258.18218994140625,
      "logps/rejected": -353.1686096191406,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.262081623077393,
      "rewards/margins": 9.589116096496582,
      "rewards/rejected": -13.8511962890625,
      "step": 1782
    },
    {
      "epoch": 2.33,
      "learning_rate": 4.4101174407110014e-05,
      "logits/chosen": -1.7826972007751465,
      "logits/rejected": -1.9188902378082275,
      "logps/chosen": -206.72190856933594,
      "logps/rejected": -328.41650390625,
      "loss": 0.0468,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.051647186279297,
      "rewards/margins": 8.272088050842285,
      "rewards/rejected": -13.323734283447266,
      "step": 1783
    },
    {
      "epoch": 2.33,
      "learning_rate": 4.409445248455399e-05,
      "logits/chosen": -1.9735987186431885,
      "logits/rejected": -2.0791704654693604,
      "logps/chosen": -252.66329956054688,
      "logps/rejected": -400.55242919921875,
      "loss": 0.0439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.900476455688477,
      "rewards/margins": 9.93791675567627,
      "rewards/rejected": -14.838394165039062,
      "step": 1784
    },
    {
      "epoch": 2.34,
      "learning_rate": 4.408772724714509e-05,
      "logits/chosen": -1.9090800285339355,
      "logits/rejected": -1.964564561843872,
      "logps/chosen": -216.352783203125,
      "logps/rejected": -327.7049560546875,
      "loss": 0.0466,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.770293712615967,
      "rewards/margins": 8.165373802185059,
      "rewards/rejected": -12.935667037963867,
      "step": 1785
    },
    {
      "epoch": 2.34,
      "learning_rate": 4.408099869605085e-05,
      "logits/chosen": -2.079120397567749,
      "logits/rejected": -2.0540542602539062,
      "logps/chosen": -324.5140075683594,
      "logps/rejected": -396.20391845703125,
      "loss": 0.0447,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.006938457489014,
      "rewards/margins": 8.714357376098633,
      "rewards/rejected": -14.721295356750488,
      "step": 1786
    },
    {
      "epoch": 2.34,
      "learning_rate": 4.4074266832439355e-05,
      "logits/chosen": -1.7295905351638794,
      "logits/rejected": -1.7185676097869873,
      "logps/chosen": -233.32333374023438,
      "logps/rejected": -330.0339050292969,
      "loss": 0.1225,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.257805347442627,
      "rewards/margins": 8.65319538116455,
      "rewards/rejected": -13.911002159118652,
      "step": 1787
    },
    {
      "epoch": 2.34,
      "learning_rate": 4.406753165747927e-05,
      "logits/chosen": -1.5425264835357666,
      "logits/rejected": -1.580695629119873,
      "logps/chosen": -213.93472290039062,
      "logps/rejected": -363.4765930175781,
      "loss": 0.0896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.49269962310791,
      "rewards/margins": 9.06192684173584,
      "rewards/rejected": -14.554627418518066,
      "step": 1788
    },
    {
      "epoch": 2.34,
      "learning_rate": 4.4060793172339855e-05,
      "logits/chosen": -1.637101173400879,
      "logits/rejected": -1.7731558084487915,
      "logps/chosen": -305.5036315917969,
      "logps/rejected": -429.70941162109375,
      "loss": 0.0523,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.886563301086426,
      "rewards/margins": 10.07012939453125,
      "rewards/rejected": -15.95669174194336,
      "step": 1789
    },
    {
      "epoch": 2.34,
      "learning_rate": 4.4054051378190915e-05,
      "logits/chosen": -1.8037257194519043,
      "logits/rejected": -1.937896728515625,
      "logps/chosen": -242.22866821289062,
      "logps/rejected": -403.0519104003906,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.685405254364014,
      "rewards/margins": 10.16978645324707,
      "rewards/rejected": -16.855192184448242,
      "step": 1790
    },
    {
      "epoch": 2.34,
      "learning_rate": 4.404730627620285e-05,
      "logits/chosen": -1.89559006690979,
      "logits/rejected": -1.9932374954223633,
      "logps/chosen": -253.1778564453125,
      "logps/rejected": -392.6427917480469,
      "loss": 0.0441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.057366847991943,
      "rewards/margins": 10.07391357421875,
      "rewards/rejected": -16.13128089904785,
      "step": 1791
    },
    {
      "epoch": 2.35,
      "learning_rate": 4.404055786754664e-05,
      "logits/chosen": -2.106956958770752,
      "logits/rejected": -2.170747995376587,
      "logps/chosen": -253.71273803710938,
      "logps/rejected": -377.1934509277344,
      "loss": 0.045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.209299087524414,
      "rewards/margins": 8.318023681640625,
      "rewards/rejected": -13.527321815490723,
      "step": 1792
    },
    {
      "epoch": 2.35,
      "learning_rate": 4.403380615339382e-05,
      "logits/chosen": -1.9337446689605713,
      "logits/rejected": -1.9458836317062378,
      "logps/chosen": -254.76597595214844,
      "logps/rejected": -364.29302978515625,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.060081958770752,
      "rewards/margins": 11.015154838562012,
      "rewards/rejected": -16.075237274169922,
      "step": 1793
    },
    {
      "epoch": 2.35,
      "learning_rate": 4.402705113491651e-05,
      "logits/chosen": -1.6449617147445679,
      "logits/rejected": -1.5874007940292358,
      "logps/chosen": -240.11338806152344,
      "logps/rejected": -294.23553466796875,
      "loss": 0.1034,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.366305351257324,
      "rewards/margins": 6.1185784339904785,
      "rewards/rejected": -11.484884262084961,
      "step": 1794
    },
    {
      "epoch": 2.35,
      "learning_rate": 4.40202928132874e-05,
      "logits/chosen": -1.9977463483810425,
      "logits/rejected": -1.9549039602279663,
      "logps/chosen": -257.6357116699219,
      "logps/rejected": -340.7739562988281,
      "loss": 0.0886,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.2195940017700195,
      "rewards/margins": 8.912274360656738,
      "rewards/rejected": -14.131868362426758,
      "step": 1795
    },
    {
      "epoch": 2.35,
      "learning_rate": 4.4013531189679747e-05,
      "logits/chosen": -1.8238599300384521,
      "logits/rejected": -1.846430778503418,
      "logps/chosen": -306.49176025390625,
      "logps/rejected": -373.8945007324219,
      "loss": 0.0444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.066048622131348,
      "rewards/margins": 9.407059669494629,
      "rewards/rejected": -15.47310733795166,
      "step": 1796
    },
    {
      "epoch": 2.35,
      "learning_rate": 4.40067662652674e-05,
      "logits/chosen": -2.069310188293457,
      "logits/rejected": -2.1010749340057373,
      "logps/chosen": -248.91537475585938,
      "logps/rejected": -339.01397705078125,
      "loss": 0.1163,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.1216020584106445,
      "rewards/margins": 9.208147048950195,
      "rewards/rejected": -14.329750061035156,
      "step": 1797
    },
    {
      "epoch": 2.35,
      "learning_rate": 4.399999804122477e-05,
      "logits/chosen": -1.7326210737228394,
      "logits/rejected": -1.844303011894226,
      "logps/chosen": -211.9335174560547,
      "logps/rejected": -317.3373107910156,
      "loss": 0.0883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.708733558654785,
      "rewards/margins": 8.05502700805664,
      "rewards/rejected": -13.763760566711426,
      "step": 1798
    },
    {
      "epoch": 2.35,
      "learning_rate": 4.399322651872683e-05,
      "logits/chosen": -1.9128247499465942,
      "logits/rejected": -1.9641433954238892,
      "logps/chosen": -238.70310974121094,
      "logps/rejected": -336.80023193359375,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.533979415893555,
      "rewards/margins": 10.12472915649414,
      "rewards/rejected": -14.658710479736328,
      "step": 1799
    },
    {
      "epoch": 2.36,
      "learning_rate": 4.398645169894914e-05,
      "logits/chosen": -1.973076343536377,
      "logits/rejected": -2.023376226425171,
      "logps/chosen": -255.61390686035156,
      "logps/rejected": -356.170654296875,
      "loss": 0.0529,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.700221538543701,
      "rewards/margins": 7.843366622924805,
      "rewards/rejected": -13.543588638305664,
      "step": 1800
    },
    {
      "epoch": 2.36,
      "learning_rate": 4.3979673583067836e-05,
      "logits/chosen": -1.8819239139556885,
      "logits/rejected": -1.7926932573318481,
      "logps/chosen": -210.09368896484375,
      "logps/rejected": -265.912353515625,
      "loss": 0.0504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.568578720092773,
      "rewards/margins": 7.491393089294434,
      "rewards/rejected": -12.059972763061523,
      "step": 1801
    },
    {
      "epoch": 2.36,
      "learning_rate": 4.3972892172259605e-05,
      "logits/chosen": -1.8808557987213135,
      "logits/rejected": -1.8767778873443604,
      "logps/chosen": -215.65005493164062,
      "logps/rejected": -278.6995849609375,
      "loss": 0.0692,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.384488105773926,
      "rewards/margins": 7.532333850860596,
      "rewards/rejected": -11.916821479797363,
      "step": 1802
    },
    {
      "epoch": 2.36,
      "learning_rate": 4.3966107467701735e-05,
      "logits/chosen": -1.7873945236206055,
      "logits/rejected": -1.922156572341919,
      "logps/chosen": -216.10459899902344,
      "logps/rejected": -328.5376281738281,
      "loss": 0.0871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.000890254974365,
      "rewards/margins": 7.771942615509033,
      "rewards/rejected": -11.772831916809082,
      "step": 1803
    },
    {
      "epoch": 2.36,
      "learning_rate": 4.3959319470572064e-05,
      "logits/chosen": -1.863304615020752,
      "logits/rejected": -1.828956961631775,
      "logps/chosen": -190.7486572265625,
      "logps/rejected": -266.3214416503906,
      "loss": 0.0772,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.7576091289520264,
      "rewards/margins": 7.365079879760742,
      "rewards/rejected": -11.122689247131348,
      "step": 1804
    },
    {
      "epoch": 2.36,
      "learning_rate": 4.3952528182049e-05,
      "logits/chosen": -2.0583343505859375,
      "logits/rejected": -2.0666375160217285,
      "logps/chosen": -203.62545776367188,
      "logps/rejected": -301.2793884277344,
      "loss": 0.0795,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.846843242645264,
      "rewards/margins": 8.276459693908691,
      "rewards/rejected": -13.123303413391113,
      "step": 1805
    },
    {
      "epoch": 2.36,
      "learning_rate": 4.394573360331155e-05,
      "logits/chosen": -2.133742332458496,
      "logits/rejected": -2.118887424468994,
      "logps/chosen": -209.5016632080078,
      "logps/rejected": -298.78485107421875,
      "loss": 0.1229,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.166501998901367,
      "rewards/margins": 7.851699352264404,
      "rewards/rejected": -13.01820182800293,
      "step": 1806
    },
    {
      "epoch": 2.36,
      "learning_rate": 4.393893573553926e-05,
      "logits/chosen": -1.800545334815979,
      "logits/rejected": -1.7241178750991821,
      "logps/chosen": -234.68531799316406,
      "logps/rejected": -313.6758117675781,
      "loss": 0.1319,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.9824187755584717,
      "rewards/margins": 8.531006813049316,
      "rewards/rejected": -12.51342487335205,
      "step": 1807
    },
    {
      "epoch": 2.37,
      "learning_rate": 4.393213457991225e-05,
      "logits/chosen": -1.7737388610839844,
      "logits/rejected": -1.7564946413040161,
      "logps/chosen": -242.74880981445312,
      "logps/rejected": -319.61334228515625,
      "loss": 0.0349,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.957961082458496,
      "rewards/margins": 9.886978149414062,
      "rewards/rejected": -14.844938278198242,
      "step": 1808
    },
    {
      "epoch": 2.37,
      "learning_rate": 4.3925330137611236e-05,
      "logits/chosen": -1.8337657451629639,
      "logits/rejected": -1.7229583263397217,
      "logps/chosen": -302.4295654296875,
      "logps/rejected": -342.783935546875,
      "loss": 0.005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.371170520782471,
      "rewards/margins": 9.544804573059082,
      "rewards/rejected": -13.915973663330078,
      "step": 1809
    },
    {
      "epoch": 2.37,
      "learning_rate": 4.391852240981749e-05,
      "logits/chosen": -1.9108226299285889,
      "logits/rejected": -2.050128936767578,
      "logps/chosen": -248.89309692382812,
      "logps/rejected": -343.4649963378906,
      "loss": 0.0439,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.320683479309082,
      "rewards/margins": 8.538372039794922,
      "rewards/rejected": -12.85905647277832,
      "step": 1810
    },
    {
      "epoch": 2.37,
      "learning_rate": 4.391171139771284e-05,
      "logits/chosen": -2.0633928775787354,
      "logits/rejected": -2.028679132461548,
      "logps/chosen": -276.5841064453125,
      "logps/rejected": -366.01812744140625,
      "loss": 0.0029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.392476558685303,
      "rewards/margins": 8.86742877960205,
      "rewards/rejected": -14.259904861450195,
      "step": 1811
    },
    {
      "epoch": 2.37,
      "learning_rate": 4.390489710247971e-05,
      "logits/chosen": -1.4684603214263916,
      "logits/rejected": -1.5528146028518677,
      "logps/chosen": -233.13583374023438,
      "logps/rejected": -316.0859680175781,
      "loss": 0.0902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.470398426055908,
      "rewards/margins": 7.3788886070251465,
      "rewards/rejected": -11.849287033081055,
      "step": 1812
    },
    {
      "epoch": 2.37,
      "learning_rate": 4.389807952530107e-05,
      "logits/chosen": -1.895721435546875,
      "logits/rejected": -1.9583930969238281,
      "logps/chosen": -232.15414428710938,
      "logps/rejected": -302.6424865722656,
      "loss": 0.0896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.65089750289917,
      "rewards/margins": 8.486051559448242,
      "rewards/rejected": -12.13694953918457,
      "step": 1813
    },
    {
      "epoch": 2.37,
      "learning_rate": 4.389125866736049e-05,
      "logits/chosen": -1.921229362487793,
      "logits/rejected": -2.0489561557769775,
      "logps/chosen": -269.0270080566406,
      "logps/rejected": -355.5064392089844,
      "loss": 0.0057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6717588901519775,
      "rewards/margins": 8.599470138549805,
      "rewards/rejected": -12.27122974395752,
      "step": 1814
    },
    {
      "epoch": 2.38,
      "learning_rate": 4.3884434529842076e-05,
      "logits/chosen": -1.8691997528076172,
      "logits/rejected": -1.7107853889465332,
      "logps/chosen": -236.80758666992188,
      "logps/rejected": -315.18115234375,
      "loss": 0.0149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.2298078536987305,
      "rewards/margins": 8.78706169128418,
      "rewards/rejected": -13.016868591308594,
      "step": 1815
    },
    {
      "epoch": 2.38,
      "learning_rate": 4.387760711393052e-05,
      "logits/chosen": -2.011821746826172,
      "logits/rejected": -2.2487993240356445,
      "logps/chosen": -233.17649841308594,
      "logps/rejected": -328.5198059082031,
      "loss": 0.0873,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.249701976776123,
      "rewards/margins": 7.541766166687012,
      "rewards/rejected": -11.791467666625977,
      "step": 1816
    },
    {
      "epoch": 2.38,
      "learning_rate": 4.3870776420811085e-05,
      "logits/chosen": -2.020697832107544,
      "logits/rejected": -2.0499825477600098,
      "logps/chosen": -172.5773468017578,
      "logps/rejected": -285.0962219238281,
      "loss": 0.0137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.833531379699707,
      "rewards/margins": 9.668582916259766,
      "rewards/rejected": -12.502114295959473,
      "step": 1817
    },
    {
      "epoch": 2.38,
      "learning_rate": 4.386394245166959e-05,
      "logits/chosen": -1.884272813796997,
      "logits/rejected": -1.8307147026062012,
      "logps/chosen": -201.13003540039062,
      "logps/rejected": -290.59246826171875,
      "loss": 0.0467,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.7439048290252686,
      "rewards/margins": 8.601780891418457,
      "rewards/rejected": -12.345686912536621,
      "step": 1818
    },
    {
      "epoch": 2.38,
      "learning_rate": 4.385710520769245e-05,
      "logits/chosen": -1.6563705205917358,
      "logits/rejected": -1.6766928434371948,
      "logps/chosen": -200.04208374023438,
      "logps/rejected": -301.2552490234375,
      "loss": 0.088,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.669889450073242,
      "rewards/margins": 8.020367622375488,
      "rewards/rejected": -12.690258026123047,
      "step": 1819
    },
    {
      "epoch": 2.38,
      "learning_rate": 4.385026469006662e-05,
      "logits/chosen": -1.7984764575958252,
      "logits/rejected": -1.9804240465164185,
      "logps/chosen": -227.8343048095703,
      "logps/rejected": -377.7736511230469,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.096676826477051,
      "rewards/margins": 10.553126335144043,
      "rewards/rejected": -14.64980411529541,
      "step": 1820
    },
    {
      "epoch": 2.38,
      "learning_rate": 4.3843420899979624e-05,
      "logits/chosen": -1.9728909730911255,
      "logits/rejected": -2.002688407897949,
      "logps/chosen": -224.78646850585938,
      "logps/rejected": -340.319091796875,
      "loss": 0.045,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.982545852661133,
      "rewards/margins": 8.316381454467773,
      "rewards/rejected": -13.298928260803223,
      "step": 1821
    },
    {
      "epoch": 2.38,
      "learning_rate": 4.3836573838619585e-05,
      "logits/chosen": -2.0015525817871094,
      "logits/rejected": -1.9768602848052979,
      "logps/chosen": -243.19590759277344,
      "logps/rejected": -351.15631103515625,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.434335708618164,
      "rewards/margins": 8.954057693481445,
      "rewards/rejected": -14.38839340209961,
      "step": 1822
    },
    {
      "epoch": 2.39,
      "learning_rate": 4.3829723507175154e-05,
      "logits/chosen": -2.051961898803711,
      "logits/rejected": -2.0231282711029053,
      "logps/chosen": -239.29270935058594,
      "logps/rejected": -332.600341796875,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.319908618927002,
      "rewards/margins": 9.003852844238281,
      "rewards/rejected": -14.323761940002441,
      "step": 1823
    },
    {
      "epoch": 2.39,
      "learning_rate": 4.382286990683558e-05,
      "logits/chosen": -1.9241172075271606,
      "logits/rejected": -1.9671472311019897,
      "logps/chosen": -254.70701599121094,
      "logps/rejected": -381.7937316894531,
      "loss": 0.0452,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.170557498931885,
      "rewards/margins": 8.524139404296875,
      "rewards/rejected": -13.694698333740234,
      "step": 1824
    },
    {
      "epoch": 2.39,
      "learning_rate": 4.381601303879066e-05,
      "logits/chosen": -1.8312177658081055,
      "logits/rejected": -1.9521081447601318,
      "logps/chosen": -256.5509338378906,
      "logps/rejected": -398.9842224121094,
      "loss": 0.0874,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.040108680725098,
      "rewards/margins": 8.639599800109863,
      "rewards/rejected": -13.679709434509277,
      "step": 1825
    },
    {
      "epoch": 2.39,
      "learning_rate": 4.3809152904230774e-05,
      "logits/chosen": -1.9518373012542725,
      "logits/rejected": -1.9363269805908203,
      "logps/chosen": -231.14834594726562,
      "logps/rejected": -355.7394104003906,
      "loss": 0.0049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.255949974060059,
      "rewards/margins": 9.791579246520996,
      "rewards/rejected": -16.047529220581055,
      "step": 1826
    },
    {
      "epoch": 2.39,
      "learning_rate": 4.380228950434686e-05,
      "logits/chosen": -2.1273605823516846,
      "logits/rejected": -2.1018548011779785,
      "logps/chosen": -242.73568725585938,
      "logps/rejected": -335.8653564453125,
      "loss": 0.12,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.828900337219238,
      "rewards/margins": 6.7826690673828125,
      "rewards/rejected": -13.61156940460205,
      "step": 1827
    },
    {
      "epoch": 2.39,
      "learning_rate": 4.379542284033043e-05,
      "logits/chosen": -2.1322808265686035,
      "logits/rejected": -2.161734104156494,
      "logps/chosen": -246.03045654296875,
      "logps/rejected": -309.78912353515625,
      "loss": 0.0295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.126348495483398,
      "rewards/margins": 9.210639953613281,
      "rewards/rejected": -13.336989402770996,
      "step": 1828
    },
    {
      "epoch": 2.39,
      "learning_rate": 4.3788552913373535e-05,
      "logits/chosen": -1.9826735258102417,
      "logits/rejected": -2.0525856018066406,
      "logps/chosen": -229.6805419921875,
      "logps/rejected": -300.07208251953125,
      "loss": 0.0483,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.896882057189941,
      "rewards/margins": 7.262111663818359,
      "rewards/rejected": -12.1589937210083,
      "step": 1829
    },
    {
      "epoch": 2.39,
      "learning_rate": 4.378167972466884e-05,
      "logits/chosen": -2.012612819671631,
      "logits/rejected": -2.0180180072784424,
      "logps/chosen": -281.0361328125,
      "logps/rejected": -357.9614562988281,
      "loss": 0.0468,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.319505214691162,
      "rewards/margins": 8.661141395568848,
      "rewards/rejected": -13.980646133422852,
      "step": 1830
    },
    {
      "epoch": 2.4,
      "learning_rate": 4.377480327540954e-05,
      "logits/chosen": -2.094277858734131,
      "logits/rejected": -2.123206615447998,
      "logps/chosen": -244.86651611328125,
      "logps/rejected": -343.1483154296875,
      "loss": 0.0076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.594821453094482,
      "rewards/margins": 8.944721221923828,
      "rewards/rejected": -14.539543151855469,
      "step": 1831
    },
    {
      "epoch": 2.4,
      "learning_rate": 4.3767923566789394e-05,
      "logits/chosen": -2.1138906478881836,
      "logits/rejected": -2.116511583328247,
      "logps/chosen": -260.0635681152344,
      "logps/rejected": -358.54510498046875,
      "loss": 0.0938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.075669288635254,
      "rewards/margins": 7.116991996765137,
      "rewards/rejected": -12.19266128540039,
      "step": 1832
    },
    {
      "epoch": 2.4,
      "learning_rate": 4.376104060000276e-05,
      "logits/chosen": -1.975137710571289,
      "logits/rejected": -1.9259370565414429,
      "logps/chosen": -246.47845458984375,
      "logps/rejected": -337.7164001464844,
      "loss": 0.1525,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.622036457061768,
      "rewards/margins": 7.510520935058594,
      "rewards/rejected": -13.132558822631836,
      "step": 1833
    },
    {
      "epoch": 2.4,
      "learning_rate": 4.375415437624454e-05,
      "logits/chosen": -1.8363233804702759,
      "logits/rejected": -1.923293113708496,
      "logps/chosen": -194.30352783203125,
      "logps/rejected": -285.00360107421875,
      "loss": 0.1239,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.983138084411621,
      "rewards/margins": 8.842789649963379,
      "rewards/rejected": -12.825925827026367,
      "step": 1834
    },
    {
      "epoch": 2.4,
      "learning_rate": 4.3747264896710185e-05,
      "logits/chosen": -2.0146749019622803,
      "logits/rejected": -2.1133487224578857,
      "logps/chosen": -230.74244689941406,
      "logps/rejected": -340.82720947265625,
      "loss": 0.0173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.959469318389893,
      "rewards/margins": 9.281587600708008,
      "rewards/rejected": -14.241056442260742,
      "step": 1835
    },
    {
      "epoch": 2.4,
      "learning_rate": 4.374037216259574e-05,
      "logits/chosen": -1.9939115047454834,
      "logits/rejected": -2.0178937911987305,
      "logps/chosen": -303.00439453125,
      "logps/rejected": -386.6276550292969,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.874594688415527,
      "rewards/margins": 8.983979225158691,
      "rewards/rejected": -14.858574867248535,
      "step": 1836
    },
    {
      "epoch": 2.4,
      "learning_rate": 4.373347617509781e-05,
      "logits/chosen": -1.9509072303771973,
      "logits/rejected": -2.012117385864258,
      "logps/chosen": -193.2029266357422,
      "logps/rejected": -295.21923828125,
      "loss": 0.0443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.532169342041016,
      "rewards/margins": 7.784310340881348,
      "rewards/rejected": -12.316478729248047,
      "step": 1837
    },
    {
      "epoch": 2.41,
      "learning_rate": 4.3726576935413554e-05,
      "logits/chosen": -2.150205373764038,
      "logits/rejected": -2.1467673778533936,
      "logps/chosen": -237.6910400390625,
      "logps/rejected": -307.8211975097656,
      "loss": 0.0046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.909461975097656,
      "rewards/margins": 8.52404499053955,
      "rewards/rejected": -13.433507919311523,
      "step": 1838
    },
    {
      "epoch": 2.41,
      "learning_rate": 4.3719674444740697e-05,
      "logits/chosen": -1.9739196300506592,
      "logits/rejected": -1.9100971221923828,
      "logps/chosen": -246.0147705078125,
      "logps/rejected": -313.3359069824219,
      "loss": 0.097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.311476707458496,
      "rewards/margins": 6.511356353759766,
      "rewards/rejected": -12.822834014892578,
      "step": 1839
    },
    {
      "epoch": 2.41,
      "learning_rate": 4.371276870427753e-05,
      "logits/chosen": -1.8190784454345703,
      "logits/rejected": -1.910037875175476,
      "logps/chosen": -184.87718200683594,
      "logps/rejected": -297.67181396484375,
      "loss": 0.0461,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.034555912017822,
      "rewards/margins": 7.538914680480957,
      "rewards/rejected": -11.573470115661621,
      "step": 1840
    },
    {
      "epoch": 2.41,
      "learning_rate": 4.370585971522291e-05,
      "logits/chosen": -1.8734077215194702,
      "logits/rejected": -1.891687035560608,
      "logps/chosen": -230.254150390625,
      "logps/rejected": -329.91033935546875,
      "loss": 0.0512,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.072037220001221,
      "rewards/margins": 8.25704574584961,
      "rewards/rejected": -13.329083442687988,
      "step": 1841
    },
    {
      "epoch": 2.41,
      "learning_rate": 4.3698947478776264e-05,
      "logits/chosen": -1.8912385702133179,
      "logits/rejected": -1.9524736404418945,
      "logps/chosen": -197.22183227539062,
      "logps/rejected": -304.8646240234375,
      "loss": 0.0283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.874556541442871,
      "rewards/margins": 7.648215293884277,
      "rewards/rejected": -11.522772789001465,
      "step": 1842
    },
    {
      "epoch": 2.41,
      "learning_rate": 4.369203199613758e-05,
      "logits/chosen": -2.2239837646484375,
      "logits/rejected": -2.220667600631714,
      "logps/chosen": -290.1179504394531,
      "logps/rejected": -369.70819091796875,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.312349319458008,
      "rewards/margins": 8.92453670501709,
      "rewards/rejected": -14.236885070800781,
      "step": 1843
    },
    {
      "epoch": 2.41,
      "learning_rate": 4.368511326850739e-05,
      "logits/chosen": -1.8601927757263184,
      "logits/rejected": -1.8277055025100708,
      "logps/chosen": -231.20823669433594,
      "logps/rejected": -274.91693115234375,
      "loss": 0.229,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.912055492401123,
      "rewards/margins": 6.686162948608398,
      "rewards/rejected": -11.598217964172363,
      "step": 1844
    },
    {
      "epoch": 2.41,
      "learning_rate": 4.367819129708682e-05,
      "logits/chosen": -2.0417680740356445,
      "logits/rejected": -2.13813853263855,
      "logps/chosen": -223.94223022460938,
      "logps/rejected": -338.69415283203125,
      "loss": 0.0757,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.077688217163086,
      "rewards/margins": 7.460683345794678,
      "rewards/rejected": -12.538371086120605,
      "step": 1845
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.367126608307753e-05,
      "logits/chosen": -1.9008760452270508,
      "logits/rejected": -2.027259111404419,
      "logps/chosen": -222.43666076660156,
      "logps/rejected": -346.3556823730469,
      "loss": 0.0308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.042816162109375,
      "rewards/margins": 9.784735679626465,
      "rewards/rejected": -13.82755184173584,
      "step": 1846
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.3664337627681774e-05,
      "logits/chosen": -2.1032116413116455,
      "logits/rejected": -2.1320929527282715,
      "logps/chosen": -204.25173950195312,
      "logps/rejected": -300.614501953125,
      "loss": 0.1124,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.063549041748047,
      "rewards/margins": 6.760867118835449,
      "rewards/rejected": -11.824416160583496,
      "step": 1847
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.365740593210234e-05,
      "logits/chosen": -2.150740146636963,
      "logits/rejected": -2.2131547927856445,
      "logps/chosen": -228.9392547607422,
      "logps/rejected": -333.38836669921875,
      "loss": 0.1739,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.91583251953125,
      "rewards/margins": 8.279642105102539,
      "rewards/rejected": -12.195475578308105,
      "step": 1848
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.36504709975426e-05,
      "logits/chosen": -1.8772621154785156,
      "logits/rejected": -1.9525573253631592,
      "logps/chosen": -310.34375,
      "logps/rejected": -355.4771728515625,
      "loss": 0.0907,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.125977993011475,
      "rewards/margins": 6.897149085998535,
      "rewards/rejected": -13.023127555847168,
      "step": 1849
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.364353282520648e-05,
      "logits/chosen": -2.17010235786438,
      "logits/rejected": -2.115602493286133,
      "logps/chosen": -233.1490936279297,
      "logps/rejected": -316.7723083496094,
      "loss": 0.0603,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.2152438163757324,
      "rewards/margins": 10.278971672058105,
      "rewards/rejected": -13.49421501159668,
      "step": 1850
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.363659141629845e-05,
      "logits/chosen": -2.1375784873962402,
      "logits/rejected": -2.1189472675323486,
      "logps/chosen": -221.9202117919922,
      "logps/rejected": -287.76275634765625,
      "loss": 0.0914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.150267601013184,
      "rewards/margins": 6.575910568237305,
      "rewards/rejected": -11.726178169250488,
      "step": 1851
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.362964677202357e-05,
      "logits/chosen": -2.344611167907715,
      "logits/rejected": -2.2970376014709473,
      "logps/chosen": -251.08851623535156,
      "logps/rejected": -334.6526794433594,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.216737270355225,
      "rewards/margins": 7.752537250518799,
      "rewards/rejected": -12.969274520874023,
      "step": 1852
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.3622698893587456e-05,
      "logits/chosen": -2.0821337699890137,
      "logits/rejected": -2.083778142929077,
      "logps/chosen": -243.42315673828125,
      "logps/rejected": -315.04339599609375,
      "loss": 0.0886,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.053452491760254,
      "rewards/margins": 8.2511625289917,
      "rewards/rejected": -12.304615020751953,
      "step": 1853
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.361574778219628e-05,
      "logits/chosen": -2.0339465141296387,
      "logits/rejected": -2.088513135910034,
      "logps/chosen": -206.04376220703125,
      "logps/rejected": -341.8985290527344,
      "loss": 0.0449,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.140441417694092,
      "rewards/margins": 9.311591148376465,
      "rewards/rejected": -14.452032089233398,
      "step": 1854
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.360879343905676e-05,
      "logits/chosen": -2.0117568969726562,
      "logits/rejected": -1.986091136932373,
      "logps/chosen": -258.1641540527344,
      "logps/rejected": -312.88116455078125,
      "loss": 0.101,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.49587345123291,
      "rewards/margins": 8.615702629089355,
      "rewards/rejected": -13.111577033996582,
      "step": 1855
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.360183586537621e-05,
      "logits/chosen": -1.5971702337265015,
      "logits/rejected": -1.8032184839248657,
      "logps/chosen": -194.74679565429688,
      "logps/rejected": -318.2879333496094,
      "loss": 0.0887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.8355278968811035,
      "rewards/margins": 7.676074504852295,
      "rewards/rejected": -13.511604309082031,
      "step": 1856
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.359487506236246e-05,
      "logits/chosen": -2.150571346282959,
      "logits/rejected": -2.0732815265655518,
      "logps/chosen": -219.47314453125,
      "logps/rejected": -299.68218994140625,
      "loss": 0.0283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4884231090545654,
      "rewards/margins": 9.733719825744629,
      "rewards/rejected": -13.22214126586914,
      "step": 1857
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.358791103122395e-05,
      "logits/chosen": -1.8017619848251343,
      "logits/rejected": -1.721257209777832,
      "logps/chosen": -204.084716796875,
      "logps/rejected": -274.8855895996094,
      "loss": 0.132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.801064968109131,
      "rewards/margins": 6.824024677276611,
      "rewards/rejected": -11.625089645385742,
      "step": 1858
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.358094377316965e-05,
      "logits/chosen": -1.9682602882385254,
      "logits/rejected": -1.961120843887329,
      "logps/chosen": -206.16127014160156,
      "logps/rejected": -274.9136657714844,
      "loss": 0.0888,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.209860801696777,
      "rewards/margins": 5.908940315246582,
      "rewards/rejected": -11.118800163269043,
      "step": 1859
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.3573973289409094e-05,
      "logits/chosen": -1.7236840724945068,
      "logits/rejected": -1.6636816263198853,
      "logps/chosen": -251.0654754638672,
      "logps/rejected": -302.43121337890625,
      "loss": 0.2175,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.596639633178711,
      "rewards/margins": 7.8836469650268555,
      "rewards/rejected": -12.480286598205566,
      "step": 1860
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.356699958115238e-05,
      "logits/chosen": -1.7644553184509277,
      "logits/rejected": -1.7401866912841797,
      "logps/chosen": -251.42462158203125,
      "logps/rejected": -335.5991516113281,
      "loss": 0.1323,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.755103588104248,
      "rewards/margins": 6.958304405212402,
      "rewards/rejected": -11.713406562805176,
      "step": 1861
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.356002264961016e-05,
      "logits/chosen": -2.147675037384033,
      "logits/rejected": -2.179133176803589,
      "logps/chosen": -230.9063262939453,
      "logps/rejected": -317.8185119628906,
      "loss": 0.0708,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.664783477783203,
      "rewards/margins": 9.451215744018555,
      "rewards/rejected": -13.115998268127441,
      "step": 1862
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.3553042495993643e-05,
      "logits/chosen": -2.1215152740478516,
      "logits/rejected": -2.17391300201416,
      "logps/chosen": -287.8443908691406,
      "logps/rejected": -335.31256103515625,
      "loss": 0.1452,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.5345940589904785,
      "rewards/margins": 5.930063247680664,
      "rewards/rejected": -10.4646577835083,
      "step": 1863
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.354605912151463e-05,
      "logits/chosen": -2.1102986335754395,
      "logits/rejected": -2.286177635192871,
      "logps/chosen": -181.72535705566406,
      "logps/rejected": -275.7710876464844,
      "loss": 0.1328,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.561611175537109,
      "rewards/margins": 7.474075794219971,
      "rewards/rejected": -12.035687446594238,
      "step": 1864
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.353907252738543e-05,
      "logits/chosen": -1.6983438730239868,
      "logits/rejected": -1.6498935222625732,
      "logps/chosen": -227.6995849609375,
      "logps/rejected": -276.0758972167969,
      "loss": 0.0712,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.649226188659668,
      "rewards/margins": 6.456134796142578,
      "rewards/rejected": -11.105360984802246,
      "step": 1865
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.353208271481894e-05,
      "logits/chosen": -2.0330216884613037,
      "logits/rejected": -2.0915470123291016,
      "logps/chosen": -237.63294982910156,
      "logps/rejected": -305.3509826660156,
      "loss": 0.0881,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.406057357788086,
      "rewards/margins": 6.762484550476074,
      "rewards/rejected": -11.168540954589844,
      "step": 1866
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.352508968502862e-05,
      "logits/chosen": -1.7718350887298584,
      "logits/rejected": -1.7792692184448242,
      "logps/chosen": -220.42466735839844,
      "logps/rejected": -275.79290771484375,
      "loss": 0.1527,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.707272529602051,
      "rewards/margins": 5.57955265045166,
      "rewards/rejected": -10.286825180053711,
      "step": 1867
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.351809343922848e-05,
      "logits/chosen": -2.1098427772521973,
      "logits/rejected": -2.210489273071289,
      "logps/chosen": -278.0294189453125,
      "logps/rejected": -352.3443908691406,
      "loss": 0.0948,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.806293487548828,
      "rewards/margins": 7.794907093048096,
      "rewards/rejected": -12.60119915008545,
      "step": 1868
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.351109397863309e-05,
      "logits/chosen": -2.126844644546509,
      "logits/rejected": -2.1765525341033936,
      "logps/chosen": -266.9029541015625,
      "logps/rejected": -331.0353088378906,
      "loss": 0.0945,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.96994686126709,
      "rewards/margins": 7.34199857711792,
      "rewards/rejected": -12.311944961547852,
      "step": 1869
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.350409130445756e-05,
      "logits/chosen": -2.012559413909912,
      "logits/rejected": -2.187136650085449,
      "logps/chosen": -217.30624389648438,
      "logps/rejected": -329.05841064453125,
      "loss": 0.0053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.945742130279541,
      "rewards/margins": 8.809176445007324,
      "rewards/rejected": -13.754919052124023,
      "step": 1870
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.34970854179176e-05,
      "logits/chosen": -2.165018081665039,
      "logits/rejected": -2.2561163902282715,
      "logps/chosen": -266.87152099609375,
      "logps/rejected": -364.1900329589844,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.06184196472168,
      "rewards/margins": 9.541501998901367,
      "rewards/rejected": -14.603343963623047,
      "step": 1871
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.349007632022943e-05,
      "logits/chosen": -1.7294069528579712,
      "logits/rejected": -1.7365280389785767,
      "logps/chosen": -224.39157104492188,
      "logps/rejected": -304.15411376953125,
      "loss": 0.0886,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.065281391143799,
      "rewards/margins": 6.858569622039795,
      "rewards/rejected": -12.923851013183594,
      "step": 1872
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.348306401260987e-05,
      "logits/chosen": -2.3519177436828613,
      "logits/rejected": -2.365565538406372,
      "logps/chosen": -260.7571716308594,
      "logps/rejected": -350.22882080078125,
      "loss": 0.0909,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.139139652252197,
      "rewards/margins": 8.925704002380371,
      "rewards/rejected": -14.064842224121094,
      "step": 1873
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.3476048496276265e-05,
      "logits/chosen": -2.2548577785491943,
      "logits/rejected": -2.293238878250122,
      "logps/chosen": -257.31414794921875,
      "logps/rejected": -358.8238525390625,
      "loss": 0.0174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.687314033508301,
      "rewards/margins": 8.911933898925781,
      "rewards/rejected": -14.599246978759766,
      "step": 1874
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.3469029772446525e-05,
      "logits/chosen": -1.6724498271942139,
      "logits/rejected": -1.61297607421875,
      "logps/chosen": -295.10333251953125,
      "logps/rejected": -342.6184997558594,
      "loss": 0.0941,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.655141830444336,
      "rewards/margins": 7.57160758972168,
      "rewards/rejected": -12.226749420166016,
      "step": 1875
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.3462007842339135e-05,
      "logits/chosen": -1.7761496305465698,
      "logits/rejected": -2.0170705318450928,
      "logps/chosen": -142.1776123046875,
      "logps/rejected": -264.1295166015625,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0851376056671143,
      "rewards/margins": 8.722681999206543,
      "rewards/rejected": -11.807819366455078,
      "step": 1876
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.345498270717311e-05,
      "logits/chosen": -1.8887841701507568,
      "logits/rejected": -1.9506733417510986,
      "logps/chosen": -242.97447204589844,
      "logps/rejected": -345.9141540527344,
      "loss": 0.1465,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.468897342681885,
      "rewards/margins": 9.694093704223633,
      "rewards/rejected": -14.162991523742676,
      "step": 1877
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.3447954368168036e-05,
      "logits/chosen": -1.8739227056503296,
      "logits/rejected": -1.8192765712738037,
      "logps/chosen": -264.2427062988281,
      "logps/rejected": -359.7630310058594,
      "loss": 0.0272,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.784821033477783,
      "rewards/margins": 7.629404544830322,
      "rewards/rejected": -13.414225578308105,
      "step": 1878
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.344092282654406e-05,
      "logits/chosen": -1.9280517101287842,
      "logits/rejected": -1.841347575187683,
      "logps/chosen": -246.22378540039062,
      "logps/rejected": -298.2712097167969,
      "loss": 0.0817,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.732930660247803,
      "rewards/margins": 7.216610908508301,
      "rewards/rejected": -11.949542045593262,
      "step": 1879
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.343388808352187e-05,
      "logits/chosen": -2.0055572986602783,
      "logits/rejected": -2.025848865509033,
      "logps/chosen": -204.11595153808594,
      "logps/rejected": -263.9717102050781,
      "loss": 0.1232,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.866933822631836,
      "rewards/margins": 6.790870189666748,
      "rewards/rejected": -11.657803535461426,
      "step": 1880
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.342685014032273e-05,
      "logits/chosen": -2.0591530799865723,
      "logits/rejected": -2.0639140605926514,
      "logps/chosen": -214.9796905517578,
      "logps/rejected": -300.53106689453125,
      "loss": 0.0562,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.563952445983887,
      "rewards/margins": 7.759393692016602,
      "rewards/rejected": -12.323345184326172,
      "step": 1881
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.341980899816843e-05,
      "logits/chosen": -1.9732942581176758,
      "logits/rejected": -2.136146306991577,
      "logps/chosen": -234.76837158203125,
      "logps/rejected": -342.0783996582031,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.5489654541015625,
      "rewards/margins": 8.756852149963379,
      "rewards/rejected": -13.305819511413574,
      "step": 1882
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.341276465828136e-05,
      "logits/chosen": -1.8425331115722656,
      "logits/rejected": -2.025358200073242,
      "logps/chosen": -227.8318634033203,
      "logps/rejected": -352.81524658203125,
      "loss": 0.0104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.6305718421936035,
      "rewards/margins": 7.565232753753662,
      "rewards/rejected": -13.19580364227295,
      "step": 1883
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.3405717121884406e-05,
      "logits/chosen": -2.106154441833496,
      "logits/rejected": -2.1269004344940186,
      "logps/chosen": -205.65060424804688,
      "logps/rejected": -257.221923828125,
      "loss": 0.1787,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.801647663116455,
      "rewards/margins": 5.70452356338501,
      "rewards/rejected": -9.506172180175781,
      "step": 1884
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.339866639020106e-05,
      "logits/chosen": -1.9552853107452393,
      "logits/rejected": -2.0579018592834473,
      "logps/chosen": -244.7666015625,
      "logps/rejected": -307.3655700683594,
      "loss": 0.014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.008973121643066,
      "rewards/margins": 7.487423896789551,
      "rewards/rejected": -11.496397972106934,
      "step": 1885
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.3391612464455344e-05,
      "logits/chosen": -1.4287627935409546,
      "logits/rejected": -1.5520751476287842,
      "logps/chosen": -212.376708984375,
      "logps/rejected": -295.0533752441406,
      "loss": 0.0884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.4821248054504395,
      "rewards/margins": 7.69401741027832,
      "rewards/rejected": -11.176141738891602,
      "step": 1886
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.3384555345871845e-05,
      "logits/chosen": -2.0539603233337402,
      "logits/rejected": -2.0998637676239014,
      "logps/chosen": -225.72454833984375,
      "logps/rejected": -307.70245361328125,
      "loss": 0.0874,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.347853183746338,
      "rewards/margins": 7.575061798095703,
      "rewards/rejected": -11.922914505004883,
      "step": 1887
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.337749503567569e-05,
      "logits/chosen": -2.240464925765991,
      "logits/rejected": -2.2448160648345947,
      "logps/chosen": -222.15724182128906,
      "logps/rejected": -313.15093994140625,
      "loss": 0.0471,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5812578201293945,
      "rewards/margins": 8.621621131896973,
      "rewards/rejected": -12.202878952026367,
      "step": 1888
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.337043153509258e-05,
      "logits/chosen": -1.7158111333847046,
      "logits/rejected": -1.7638497352600098,
      "logps/chosen": -295.1845397949219,
      "logps/rejected": -395.8709411621094,
      "loss": 0.0444,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.137034893035889,
      "rewards/margins": 8.628808975219727,
      "rewards/rejected": -12.765844345092773,
      "step": 1889
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.3363364845348755e-05,
      "logits/chosen": -1.9096884727478027,
      "logits/rejected": -1.9671432971954346,
      "logps/chosen": -214.6175537109375,
      "logps/rejected": -330.2419738769531,
      "loss": 0.0323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.5655999183654785,
      "rewards/margins": 8.131664276123047,
      "rewards/rejected": -12.697264671325684,
      "step": 1890
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.335629496767101e-05,
      "logits/chosen": -2.1084601879119873,
      "logits/rejected": -2.018869400024414,
      "logps/chosen": -228.5397186279297,
      "logps/rejected": -305.6234130859375,
      "loss": 0.0035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.8502936363220215,
      "rewards/margins": 8.437298774719238,
      "rewards/rejected": -12.287591934204102,
      "step": 1891
    },
    {
      "epoch": 2.48,
      "learning_rate": 4.334922190328671e-05,
      "logits/chosen": -2.083664894104004,
      "logits/rejected": -2.084148406982422,
      "logps/chosen": -226.5367431640625,
      "logps/rejected": -316.456298828125,
      "loss": 0.0627,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.226537227630615,
      "rewards/margins": 6.392550468444824,
      "rewards/rejected": -11.619088172912598,
      "step": 1892
    },
    {
      "epoch": 2.48,
      "learning_rate": 4.3342145653423736e-05,
      "logits/chosen": -2.1525473594665527,
      "logits/rejected": -2.1263246536254883,
      "logps/chosen": -282.1151123046875,
      "logps/rejected": -358.0589599609375,
      "loss": 0.0451,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.356586456298828,
      "rewards/margins": 8.507760047912598,
      "rewards/rejected": -11.86434555053711,
      "step": 1893
    },
    {
      "epoch": 2.48,
      "learning_rate": 4.3335066219310564e-05,
      "logits/chosen": -2.1275317668914795,
      "logits/rejected": -2.2363266944885254,
      "logps/chosen": -217.88424682617188,
      "logps/rejected": -314.2412109375,
      "loss": 0.012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.613062858581543,
      "rewards/margins": 7.576802730560303,
      "rewards/rejected": -12.18986701965332,
      "step": 1894
    },
    {
      "epoch": 2.48,
      "learning_rate": 4.3327983602176196e-05,
      "logits/chosen": -2.2973012924194336,
      "logits/rejected": -2.352452516555786,
      "logps/chosen": -258.4927978515625,
      "logps/rejected": -352.5173034667969,
      "loss": 0.0449,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.170706748962402,
      "rewards/margins": 8.284988403320312,
      "rewards/rejected": -12.455696105957031,
      "step": 1895
    },
    {
      "epoch": 2.48,
      "learning_rate": 4.3320897803250206e-05,
      "logits/chosen": -2.1969974040985107,
      "logits/rejected": -2.2443318367004395,
      "logps/chosen": -283.5622863769531,
      "logps/rejected": -373.3434143066406,
      "loss": 0.0531,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.566214561462402,
      "rewards/margins": 8.141165733337402,
      "rewards/rejected": -12.707380294799805,
      "step": 1896
    },
    {
      "epoch": 2.48,
      "learning_rate": 4.331380882376269e-05,
      "logits/chosen": -1.893761157989502,
      "logits/rejected": -2.058621644973755,
      "logps/chosen": -239.73043823242188,
      "logps/rejected": -331.2525939941406,
      "loss": 0.0058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.567351341247559,
      "rewards/margins": 7.453845024108887,
      "rewards/rejected": -12.021196365356445,
      "step": 1897
    },
    {
      "epoch": 2.48,
      "learning_rate": 4.3306716664944344e-05,
      "logits/chosen": -2.0633487701416016,
      "logits/rejected": -2.200991153717041,
      "logps/chosen": -208.24517822265625,
      "logps/rejected": -329.59649658203125,
      "loss": 0.1383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.751850128173828,
      "rewards/margins": 6.950207233428955,
      "rewards/rejected": -11.702056884765625,
      "step": 1898
    },
    {
      "epoch": 2.49,
      "learning_rate": 4.329962132802636e-05,
      "logits/chosen": -2.092829704284668,
      "logits/rejected": -2.0219295024871826,
      "logps/chosen": -235.95083618164062,
      "logps/rejected": -304.3815002441406,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.290584564208984,
      "rewards/margins": 8.294286727905273,
      "rewards/rejected": -12.584871292114258,
      "step": 1899
    },
    {
      "epoch": 2.49,
      "learning_rate": 4.329252281424052e-05,
      "logits/chosen": -2.03996205329895,
      "logits/rejected": -2.141855239868164,
      "logps/chosen": -238.59591674804688,
      "logps/rejected": -335.9581298828125,
      "loss": 0.0167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.734767913818359,
      "rewards/margins": 8.400701522827148,
      "rewards/rejected": -13.135469436645508,
      "step": 1900
    },
    {
      "epoch": 2.49,
      "learning_rate": 4.328542112481915e-05,
      "logits/chosen": -2.228651523590088,
      "logits/rejected": -2.1372406482696533,
      "logps/chosen": -215.5223388671875,
      "logps/rejected": -306.9183044433594,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.169659614562988,
      "rewards/margins": 10.764291763305664,
      "rewards/rejected": -14.933952331542969,
      "step": 1901
    },
    {
      "epoch": 2.49,
      "learning_rate": 4.327831626099512e-05,
      "logits/chosen": -2.0305263996124268,
      "logits/rejected": -2.1352574825286865,
      "logps/chosen": -200.35475158691406,
      "logps/rejected": -329.01995849609375,
      "loss": 0.0539,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.779258728027344,
      "rewards/margins": 7.56350040435791,
      "rewards/rejected": -12.342758178710938,
      "step": 1902
    },
    {
      "epoch": 2.49,
      "learning_rate": 4.3271208224001856e-05,
      "logits/chosen": -2.0742321014404297,
      "logits/rejected": -2.106760263442993,
      "logps/chosen": -312.0980529785156,
      "logps/rejected": -396.3582458496094,
      "loss": 0.0448,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.070708274841309,
      "rewards/margins": 7.27605676651001,
      "rewards/rejected": -13.346765518188477,
      "step": 1903
    },
    {
      "epoch": 2.49,
      "learning_rate": 4.3264097015073334e-05,
      "logits/chosen": -1.9420523643493652,
      "logits/rejected": -2.0621085166931152,
      "logps/chosen": -238.076904296875,
      "logps/rejected": -330.4910888671875,
      "loss": 0.0938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.611868381500244,
      "rewards/margins": 6.453281879425049,
      "rewards/rejected": -12.06515121459961,
      "step": 1904
    },
    {
      "epoch": 2.49,
      "learning_rate": 4.325698263544409e-05,
      "logits/chosen": -1.7279473543167114,
      "logits/rejected": -1.8486162424087524,
      "logps/chosen": -156.3711700439453,
      "logps/rejected": -285.4295349121094,
      "loss": 0.09,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.322530746459961,
      "rewards/margins": 9.202067375183105,
      "rewards/rejected": -12.52459716796875,
      "step": 1905
    },
    {
      "epoch": 2.49,
      "learning_rate": 4.3249865086349186e-05,
      "logits/chosen": -1.7305097579956055,
      "logits/rejected": -1.7878034114837646,
      "logps/chosen": -225.58343505859375,
      "logps/rejected": -343.70068359375,
      "loss": 0.0461,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.001452922821045,
      "rewards/margins": 8.839422225952148,
      "rewards/rejected": -12.840875625610352,
      "step": 1906
    },
    {
      "epoch": 2.5,
      "learning_rate": 4.324274436902425e-05,
      "logits/chosen": -2.0604379177093506,
      "logits/rejected": -2.040114641189575,
      "logps/chosen": -216.70742797851562,
      "logps/rejected": -292.2000427246094,
      "loss": 0.0883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.9439444541931152,
      "rewards/margins": 7.755221843719482,
      "rewards/rejected": -11.699165344238281,
      "step": 1907
    },
    {
      "epoch": 2.5,
      "learning_rate": 4.323562048470547e-05,
      "logits/chosen": -1.9717481136322021,
      "logits/rejected": -1.9101167917251587,
      "logps/chosen": -255.74168395996094,
      "logps/rejected": -342.4549560546875,
      "loss": 0.0031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.992917537689209,
      "rewards/margins": 9.150410652160645,
      "rewards/rejected": -13.143327713012695,
      "step": 1908
    },
    {
      "epoch": 2.5,
      "learning_rate": 4.322849343462957e-05,
      "logits/chosen": -2.0949337482452393,
      "logits/rejected": -2.132563591003418,
      "logps/chosen": -266.8226013183594,
      "logps/rejected": -333.36676025390625,
      "loss": 0.0466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.620141983032227,
      "rewards/margins": 7.952317237854004,
      "rewards/rejected": -13.572460174560547,
      "step": 1909
    },
    {
      "epoch": 2.5,
      "learning_rate": 4.3221363220033814e-05,
      "logits/chosen": -1.9663276672363281,
      "logits/rejected": -1.9690686464309692,
      "logps/chosen": -256.2469482421875,
      "logps/rejected": -334.94122314453125,
      "loss": 0.1329,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.505515098571777,
      "rewards/margins": 8.074156761169434,
      "rewards/rejected": -12.579671859741211,
      "step": 1910
    },
    {
      "epoch": 2.5,
      "learning_rate": 4.321422984215605e-05,
      "logits/chosen": -1.745408058166504,
      "logits/rejected": -1.7534682750701904,
      "logps/chosen": -214.59886169433594,
      "logps/rejected": -293.2696533203125,
      "loss": 0.0908,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.432751178741455,
      "rewards/margins": 6.949195384979248,
      "rewards/rejected": -11.381946563720703,
      "step": 1911
    },
    {
      "epoch": 2.5,
      "learning_rate": 4.320709330223464e-05,
      "logits/chosen": -1.8495633602142334,
      "logits/rejected": -1.7307114601135254,
      "logps/chosen": -282.5513610839844,
      "logps/rejected": -316.41162109375,
      "loss": 0.1304,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.877445697784424,
      "rewards/margins": 10.04818344116211,
      "rewards/rejected": -13.925628662109375,
      "step": 1912
    },
    {
      "epoch": 2.5,
      "learning_rate": 4.319995360150851e-05,
      "logits/chosen": -2.143476963043213,
      "logits/rejected": -2.24068021774292,
      "logps/chosen": -257.17938232421875,
      "logps/rejected": -412.9932556152344,
      "loss": 0.0023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.908987522125244,
      "rewards/margins": 9.167608261108398,
      "rewards/rejected": -15.076594352722168,
      "step": 1913
    },
    {
      "epoch": 2.5,
      "learning_rate": 4.319281074121714e-05,
      "logits/chosen": -2.158090829849243,
      "logits/rejected": -2.0695207118988037,
      "logps/chosen": -242.647705078125,
      "logps/rejected": -316.803955078125,
      "loss": 0.0452,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.062627792358398,
      "rewards/margins": 8.320158004760742,
      "rewards/rejected": -13.38278579711914,
      "step": 1914
    },
    {
      "epoch": 2.51,
      "learning_rate": 4.318566472260054e-05,
      "logits/chosen": -1.846653699874878,
      "logits/rejected": -1.94084632396698,
      "logps/chosen": -207.33084106445312,
      "logps/rejected": -337.37139892578125,
      "loss": 0.0906,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.298641681671143,
      "rewards/margins": 7.939152240753174,
      "rewards/rejected": -12.237793922424316,
      "step": 1915
    },
    {
      "epoch": 2.51,
      "learning_rate": 4.317851554689928e-05,
      "logits/chosen": -1.935530185699463,
      "logits/rejected": -1.9438458681106567,
      "logps/chosen": -217.8266143798828,
      "logps/rejected": -332.8846130371094,
      "loss": 0.0205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.889610767364502,
      "rewards/margins": 10.1079683303833,
      "rewards/rejected": -14.997577667236328,
      "step": 1916
    },
    {
      "epoch": 2.51,
      "learning_rate": 4.3171363215354485e-05,
      "logits/chosen": -2.0048739910125732,
      "logits/rejected": -2.141301155090332,
      "logps/chosen": -208.8019256591797,
      "logps/rejected": -275.9781799316406,
      "loss": 0.0511,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.808550834655762,
      "rewards/margins": 6.669939041137695,
      "rewards/rejected": -12.478490829467773,
      "step": 1917
    },
    {
      "epoch": 2.51,
      "learning_rate": 4.316420772920782e-05,
      "logits/chosen": -1.9847973585128784,
      "logits/rejected": -2.108638286590576,
      "logps/chosen": -181.55844116210938,
      "logps/rejected": -295.7936706542969,
      "loss": 0.0444,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.9625251293182373,
      "rewards/margins": 8.532015800476074,
      "rewards/rejected": -12.494542121887207,
      "step": 1918
    },
    {
      "epoch": 2.51,
      "learning_rate": 4.3157049089701495e-05,
      "logits/chosen": -2.231801748275757,
      "logits/rejected": -2.2912943363189697,
      "logps/chosen": -291.9457702636719,
      "logps/rejected": -368.4671325683594,
      "loss": 0.0041,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.943848133087158,
      "rewards/margins": 7.637473106384277,
      "rewards/rejected": -13.581321716308594,
      "step": 1919
    },
    {
      "epoch": 2.51,
      "learning_rate": 4.3149887298078276e-05,
      "logits/chosen": -1.9980155229568481,
      "logits/rejected": -1.9702084064483643,
      "logps/chosen": -221.4971160888672,
      "logps/rejected": -364.04388427734375,
      "loss": 0.047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.686988830566406,
      "rewards/margins": 6.865217685699463,
      "rewards/rejected": -12.552206993103027,
      "step": 1920
    },
    {
      "epoch": 2.51,
      "learning_rate": 4.3142722355581466e-05,
      "logits/chosen": -1.8039441108703613,
      "logits/rejected": -1.716475009918213,
      "logps/chosen": -250.4287872314453,
      "logps/rejected": -317.84368896484375,
      "loss": 0.1178,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.042150497436523,
      "rewards/margins": 7.001775741577148,
      "rewards/rejected": -12.043927192687988,
      "step": 1921
    },
    {
      "epoch": 2.52,
      "learning_rate": 4.313555426345492e-05,
      "logits/chosen": -1.9541935920715332,
      "logits/rejected": -2.0513226985931396,
      "logps/chosen": -260.1856689453125,
      "logps/rejected": -363.9661560058594,
      "loss": 0.0913,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.970685958862305,
      "rewards/margins": 8.336792945861816,
      "rewards/rejected": -13.307477951049805,
      "step": 1922
    },
    {
      "epoch": 2.52,
      "learning_rate": 4.3128383022943045e-05,
      "logits/chosen": -2.263624906539917,
      "logits/rejected": -2.3434362411499023,
      "logps/chosen": -217.94793701171875,
      "logps/rejected": -312.8447265625,
      "loss": 0.0872,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.745199680328369,
      "rewards/margins": 7.728671550750732,
      "rewards/rejected": -11.473870277404785,
      "step": 1923
    },
    {
      "epoch": 2.52,
      "learning_rate": 4.3121208635290775e-05,
      "logits/chosen": -2.2434561252593994,
      "logits/rejected": -2.2021243572235107,
      "logps/chosen": -271.01287841796875,
      "logps/rejected": -342.8251037597656,
      "loss": 0.0479,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.077291488647461,
      "rewards/margins": 7.209915637969971,
      "rewards/rejected": -12.28720760345459,
      "step": 1924
    },
    {
      "epoch": 2.52,
      "learning_rate": 4.3114031101743625e-05,
      "logits/chosen": -2.0379481315612793,
      "logits/rejected": -2.0702476501464844,
      "logps/chosen": -255.35540771484375,
      "logps/rejected": -362.4584045410156,
      "loss": 0.0079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.8948774337768555,
      "rewards/margins": 7.793493747711182,
      "rewards/rejected": -13.688371658325195,
      "step": 1925
    },
    {
      "epoch": 2.52,
      "learning_rate": 4.3106850423547626e-05,
      "logits/chosen": -2.2296431064605713,
      "logits/rejected": -2.2409510612487793,
      "logps/chosen": -271.7919921875,
      "logps/rejected": -329.9810791015625,
      "loss": 0.0459,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.642647743225098,
      "rewards/margins": 8.106131553649902,
      "rewards/rejected": -13.748778343200684,
      "step": 1926
    },
    {
      "epoch": 2.52,
      "learning_rate": 4.309966660194936e-05,
      "logits/chosen": -2.1193394660949707,
      "logits/rejected": -2.2471766471862793,
      "logps/chosen": -233.57217407226562,
      "logps/rejected": -302.92852783203125,
      "loss": 0.0997,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.8353095054626465,
      "rewards/margins": 7.02433967590332,
      "rewards/rejected": -12.859648704528809,
      "step": 1927
    },
    {
      "epoch": 2.52,
      "learning_rate": 4.309247963819597e-05,
      "logits/chosen": -2.1643905639648438,
      "logits/rejected": -2.1485488414764404,
      "logps/chosen": -219.8082733154297,
      "logps/rejected": -281.7580261230469,
      "loss": 0.1519,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.9344429969787598,
      "rewards/margins": 6.320281982421875,
      "rewards/rejected": -10.254724502563477,
      "step": 1928
    },
    {
      "epoch": 2.52,
      "learning_rate": 4.3085289533535126e-05,
      "logits/chosen": -2.2482452392578125,
      "logits/rejected": -2.1758103370666504,
      "logps/chosen": -306.8867492675781,
      "logps/rejected": -343.5700988769531,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.638154983520508,
      "rewards/margins": 8.893589973449707,
      "rewards/rejected": -13.531745910644531,
      "step": 1929
    },
    {
      "epoch": 2.53,
      "learning_rate": 4.307809628921505e-05,
      "logits/chosen": -2.1660923957824707,
      "logits/rejected": -2.193176507949829,
      "logps/chosen": -303.06817626953125,
      "logps/rejected": -382.7147216796875,
      "loss": 0.0022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.131650447845459,
      "rewards/margins": 8.652751922607422,
      "rewards/rejected": -14.784402847290039,
      "step": 1930
    },
    {
      "epoch": 2.53,
      "learning_rate": 4.3070899906484515e-05,
      "logits/chosen": -1.7756067514419556,
      "logits/rejected": -1.7884242534637451,
      "logps/chosen": -237.867919921875,
      "logps/rejected": -322.68572998046875,
      "loss": 0.0874,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.668308258056641,
      "rewards/margins": 9.159331321716309,
      "rewards/rejected": -13.827640533447266,
      "step": 1931
    },
    {
      "epoch": 2.53,
      "learning_rate": 4.306370038659283e-05,
      "logits/chosen": -1.9823037385940552,
      "logits/rejected": -1.8524787425994873,
      "logps/chosen": -227.22921752929688,
      "logps/rejected": -342.2689208984375,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.744987964630127,
      "rewards/margins": 10.096012115478516,
      "rewards/rejected": -14.840999603271484,
      "step": 1932
    },
    {
      "epoch": 2.53,
      "learning_rate": 4.305649773078987e-05,
      "logits/chosen": -1.9377398490905762,
      "logits/rejected": -1.9592432975769043,
      "logps/chosen": -294.3237609863281,
      "logps/rejected": -344.45953369140625,
      "loss": 0.0617,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.796104431152344,
      "rewards/margins": 8.316561698913574,
      "rewards/rejected": -13.112667083740234,
      "step": 1933
    },
    {
      "epoch": 2.53,
      "learning_rate": 4.3049291940326007e-05,
      "logits/chosen": -2.130969762802124,
      "logits/rejected": -2.2177491188049316,
      "logps/chosen": -216.68133544921875,
      "logps/rejected": -305.097900390625,
      "loss": 0.0447,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.305083274841309,
      "rewards/margins": 7.978094100952148,
      "rewards/rejected": -13.283178329467773,
      "step": 1934
    },
    {
      "epoch": 2.53,
      "learning_rate": 4.3042083016452205e-05,
      "logits/chosen": -2.074617624282837,
      "logits/rejected": -2.2133822441101074,
      "logps/chosen": -206.17091369628906,
      "logps/rejected": -344.6942138671875,
      "loss": 0.0455,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.658586502075195,
      "rewards/margins": 10.279059410095215,
      "rewards/rejected": -14.93764591217041,
      "step": 1935
    },
    {
      "epoch": 2.53,
      "learning_rate": 4.303487096041996e-05,
      "logits/chosen": -2.004681348800659,
      "logits/rejected": -2.045778751373291,
      "logps/chosen": -205.9289093017578,
      "logps/rejected": -282.0126953125,
      "loss": 0.054,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.651937484741211,
      "rewards/margins": 9.119819641113281,
      "rewards/rejected": -12.771757125854492,
      "step": 1936
    },
    {
      "epoch": 2.53,
      "learning_rate": 4.3027655773481294e-05,
      "logits/chosen": -2.2388157844543457,
      "logits/rejected": -2.2951581478118896,
      "logps/chosen": -220.75875854492188,
      "logps/rejected": -293.530029296875,
      "loss": 0.048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.052151679992676,
      "rewards/margins": 7.8823113441467285,
      "rewards/rejected": -12.934463500976562,
      "step": 1937
    },
    {
      "epoch": 2.54,
      "learning_rate": 4.302043745688879e-05,
      "logits/chosen": -1.7866822481155396,
      "logits/rejected": -1.8886862993240356,
      "logps/chosen": -230.29017639160156,
      "logps/rejected": -349.378173828125,
      "loss": 0.1366,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.138594150543213,
      "rewards/margins": 8.649319648742676,
      "rewards/rejected": -13.78791332244873,
      "step": 1938
    },
    {
      "epoch": 2.54,
      "learning_rate": 4.3013216011895575e-05,
      "logits/chosen": -1.9889922142028809,
      "logits/rejected": -1.9763219356536865,
      "logps/chosen": -223.85325622558594,
      "logps/rejected": -302.8853759765625,
      "loss": 0.0452,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.246161937713623,
      "rewards/margins": 8.81850814819336,
      "rewards/rejected": -14.064668655395508,
      "step": 1939
    },
    {
      "epoch": 2.54,
      "learning_rate": 4.300599143975529e-05,
      "logits/chosen": -1.5978270769119263,
      "logits/rejected": -1.6645034551620483,
      "logps/chosen": -237.7843780517578,
      "logps/rejected": -333.2315368652344,
      "loss": 0.0893,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.7656989097595215,
      "rewards/margins": 7.666550636291504,
      "rewards/rejected": -13.432250022888184,
      "step": 1940
    },
    {
      "epoch": 2.54,
      "learning_rate": 4.299876374172217e-05,
      "logits/chosen": -2.169827699661255,
      "logits/rejected": -2.2364706993103027,
      "logps/chosen": -246.08718872070312,
      "logps/rejected": -326.4261169433594,
      "loss": 0.0677,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.478606224060059,
      "rewards/margins": 8.510499000549316,
      "rewards/rejected": -12.989105224609375,
      "step": 1941
    },
    {
      "epoch": 2.54,
      "learning_rate": 4.2991532919050945e-05,
      "logits/chosen": -1.9586464166641235,
      "logits/rejected": -1.9860866069793701,
      "logps/chosen": -219.77731323242188,
      "logps/rejected": -288.02972412109375,
      "loss": 0.056,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.910635948181152,
      "rewards/margins": 7.954824447631836,
      "rewards/rejected": -12.865461349487305,
      "step": 1942
    },
    {
      "epoch": 2.54,
      "learning_rate": 4.298429897299692e-05,
      "logits/chosen": -2.0101494789123535,
      "logits/rejected": -2.1010165214538574,
      "logps/chosen": -252.91891479492188,
      "logps/rejected": -373.2341613769531,
      "loss": 0.0096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.272066593170166,
      "rewards/margins": 8.000301361083984,
      "rewards/rejected": -14.272368431091309,
      "step": 1943
    },
    {
      "epoch": 2.54,
      "learning_rate": 4.297706190481592e-05,
      "logits/chosen": -2.028812885284424,
      "logits/rejected": -2.1959941387176514,
      "logps/chosen": -293.4753112792969,
      "logps/rejected": -407.5643615722656,
      "loss": 0.0575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.024116516113281,
      "rewards/margins": 7.825641632080078,
      "rewards/rejected": -13.84975814819336,
      "step": 1944
    },
    {
      "epoch": 2.55,
      "learning_rate": 4.2969821715764335e-05,
      "logits/chosen": -1.7866820096969604,
      "logits/rejected": -1.804237723350525,
      "logps/chosen": -227.78514099121094,
      "logps/rejected": -305.8616943359375,
      "loss": 0.0882,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.905126094818115,
      "rewards/margins": 8.556636810302734,
      "rewards/rejected": -13.461762428283691,
      "step": 1945
    },
    {
      "epoch": 2.55,
      "learning_rate": 4.2962578407099065e-05,
      "logits/chosen": -1.9624371528625488,
      "logits/rejected": -2.035775661468506,
      "logps/chosen": -218.71646118164062,
      "logps/rejected": -313.052734375,
      "loss": 0.0134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.000354766845703,
      "rewards/margins": 9.645698547363281,
      "rewards/rejected": -14.6460542678833,
      "step": 1946
    },
    {
      "epoch": 2.55,
      "learning_rate": 4.2955331980077574e-05,
      "logits/chosen": -1.8719345331192017,
      "logits/rejected": -2.02571177482605,
      "logps/chosen": -217.122314453125,
      "logps/rejected": -345.24407958984375,
      "loss": 0.044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.4896368980407715,
      "rewards/margins": 9.622235298156738,
      "rewards/rejected": -15.111871719360352,
      "step": 1947
    },
    {
      "epoch": 2.55,
      "learning_rate": 4.294808243595787e-05,
      "logits/chosen": -2.0347461700439453,
      "logits/rejected": -2.038332939147949,
      "logps/chosen": -234.9548797607422,
      "logps/rejected": -350.6724853515625,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.863475799560547,
      "rewards/margins": 11.791455268859863,
      "rewards/rejected": -16.654932022094727,
      "step": 1948
    },
    {
      "epoch": 2.55,
      "learning_rate": 4.2940829775998494e-05,
      "logits/chosen": -1.955130934715271,
      "logits/rejected": -1.9799152612686157,
      "logps/chosen": -293.515869140625,
      "logps/rejected": -395.7729187011719,
      "loss": 0.1102,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.166569709777832,
      "rewards/margins": 8.504650115966797,
      "rewards/rejected": -14.671219825744629,
      "step": 1949
    },
    {
      "epoch": 2.55,
      "learning_rate": 4.2933574001458524e-05,
      "logits/chosen": -1.9489941596984863,
      "logits/rejected": -1.9857006072998047,
      "logps/chosen": -256.5664367675781,
      "logps/rejected": -341.5423583984375,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.8608479499816895,
      "rewards/margins": 10.079227447509766,
      "rewards/rejected": -14.94007682800293,
      "step": 1950
    },
    {
      "epoch": 2.55,
      "learning_rate": 4.2926315113597585e-05,
      "logits/chosen": -1.9592641592025757,
      "logits/rejected": -1.9216690063476562,
      "logps/chosen": -266.249755859375,
      "logps/rejected": -317.3481750488281,
      "loss": 0.039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.881652355194092,
      "rewards/margins": 8.534133911132812,
      "rewards/rejected": -13.41578483581543,
      "step": 1951
    },
    {
      "epoch": 2.55,
      "learning_rate": 4.2919053113675845e-05,
      "logits/chosen": -1.9309496879577637,
      "logits/rejected": -1.9673490524291992,
      "logps/chosen": -243.59275817871094,
      "logps/rejected": -375.9563903808594,
      "loss": 0.0044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.986482620239258,
      "rewards/margins": 9.081639289855957,
      "rewards/rejected": -14.068122863769531,
      "step": 1952
    },
    {
      "epoch": 2.56,
      "learning_rate": 4.2911788002954e-05,
      "logits/chosen": -2.217352867126465,
      "logits/rejected": -2.2447142601013184,
      "logps/chosen": -230.89761352539062,
      "logps/rejected": -357.30499267578125,
      "loss": 0.0028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.1550750732421875,
      "rewards/margins": 8.492586135864258,
      "rewards/rejected": -13.647661209106445,
      "step": 1953
    },
    {
      "epoch": 2.56,
      "learning_rate": 4.290451978269331e-05,
      "logits/chosen": -2.000579833984375,
      "logits/rejected": -2.112433910369873,
      "logps/chosen": -211.8125457763672,
      "logps/rejected": -282.7967224121094,
      "loss": 0.1571,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.382208824157715,
      "rewards/margins": 7.275820732116699,
      "rewards/rejected": -11.658029556274414,
      "step": 1954
    },
    {
      "epoch": 2.56,
      "learning_rate": 4.2897248454155544e-05,
      "logits/chosen": -1.8008928298950195,
      "logits/rejected": -1.8354158401489258,
      "logps/chosen": -209.97618103027344,
      "logps/rejected": -256.56048583984375,
      "loss": 0.1602,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.440002679824829,
      "rewards/margins": 7.413758754730225,
      "rewards/rejected": -10.853761672973633,
      "step": 1955
    },
    {
      "epoch": 2.56,
      "learning_rate": 4.288997401860303e-05,
      "logits/chosen": -2.3489272594451904,
      "logits/rejected": -2.386540174484253,
      "logps/chosen": -252.65460205078125,
      "logps/rejected": -392.34906005859375,
      "loss": 0.0468,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.8349080085754395,
      "rewards/margins": 8.861000061035156,
      "rewards/rejected": -14.695908546447754,
      "step": 1956
    },
    {
      "epoch": 2.56,
      "learning_rate": 4.288269647729863e-05,
      "logits/chosen": -2.171231985092163,
      "logits/rejected": -2.142423391342163,
      "logps/chosen": -292.1837158203125,
      "logps/rejected": -372.4999694824219,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.801056861877441,
      "rewards/margins": 9.458931922912598,
      "rewards/rejected": -14.259988784790039,
      "step": 1957
    },
    {
      "epoch": 2.56,
      "learning_rate": 4.287541583150576e-05,
      "logits/chosen": -1.8236223459243774,
      "logits/rejected": -1.9413220882415771,
      "logps/chosen": -229.91751098632812,
      "logps/rejected": -327.6510314941406,
      "loss": 0.0887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.326351165771484,
      "rewards/margins": 8.002523422241211,
      "rewards/rejected": -13.328875541687012,
      "step": 1958
    },
    {
      "epoch": 2.56,
      "learning_rate": 4.286813208248834e-05,
      "logits/chosen": -1.8356335163116455,
      "logits/rejected": -1.9949692487716675,
      "logps/chosen": -212.74758911132812,
      "logps/rejected": -322.6806640625,
      "loss": 0.0663,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.013042449951172,
      "rewards/margins": 8.232125282287598,
      "rewards/rejected": -13.245166778564453,
      "step": 1959
    },
    {
      "epoch": 2.57,
      "learning_rate": 4.2860845231510873e-05,
      "logits/chosen": -2.0356662273406982,
      "logits/rejected": -1.986034870147705,
      "logps/chosen": -189.760498046875,
      "logps/rejected": -233.2946014404297,
      "loss": 0.0669,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.721360683441162,
      "rewards/margins": 5.633640766143799,
      "rewards/rejected": -10.355001449584961,
      "step": 1960
    },
    {
      "epoch": 2.57,
      "learning_rate": 4.285355527983836e-05,
      "logits/chosen": -2.2257447242736816,
      "logits/rejected": -2.207811117172241,
      "logps/chosen": -252.7733154296875,
      "logps/rejected": -355.315673828125,
      "loss": 0.0071,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.237861156463623,
      "rewards/margins": 7.754915237426758,
      "rewards/rejected": -12.992775917053223,
      "step": 1961
    },
    {
      "epoch": 2.57,
      "learning_rate": 4.2846262228736354e-05,
      "logits/chosen": -1.7899184226989746,
      "logits/rejected": -1.8761553764343262,
      "logps/chosen": -227.38531494140625,
      "logps/rejected": -309.0900573730469,
      "loss": 0.0556,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.205245018005371,
      "rewards/margins": 7.232666015625,
      "rewards/rejected": -12.437909126281738,
      "step": 1962
    },
    {
      "epoch": 2.57,
      "learning_rate": 4.283896607947098e-05,
      "logits/chosen": -1.791359782218933,
      "logits/rejected": -1.85542631149292,
      "logps/chosen": -215.9515838623047,
      "logps/rejected": -274.5942687988281,
      "loss": 0.0975,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.920288562774658,
      "rewards/margins": 6.694026947021484,
      "rewards/rejected": -11.614316940307617,
      "step": 1963
    },
    {
      "epoch": 2.57,
      "learning_rate": 4.283166683330883e-05,
      "logits/chosen": -1.8729884624481201,
      "logits/rejected": -2.0725789070129395,
      "logps/chosen": -237.73728942871094,
      "logps/rejected": -363.7380065917969,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.638874053955078,
      "rewards/margins": 9.738240242004395,
      "rewards/rejected": -15.377115249633789,
      "step": 1964
    },
    {
      "epoch": 2.57,
      "learning_rate": 4.282436449151711e-05,
      "logits/chosen": -1.958586573600769,
      "logits/rejected": -1.9482567310333252,
      "logps/chosen": -311.7152404785156,
      "logps/rejected": -435.96923828125,
      "loss": 0.1322,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.224272727966309,
      "rewards/margins": 6.595800399780273,
      "rewards/rejected": -12.820073127746582,
      "step": 1965
    },
    {
      "epoch": 2.57,
      "learning_rate": 4.2817059055363505e-05,
      "logits/chosen": -1.7612496614456177,
      "logits/rejected": -1.7791756391525269,
      "logps/chosen": -194.8032684326172,
      "logps/rejected": -292.1495666503906,
      "loss": 0.2856,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.353596210479736,
      "rewards/margins": 5.999757766723633,
      "rewards/rejected": -11.353353500366211,
      "step": 1966
    },
    {
      "epoch": 2.57,
      "learning_rate": 4.280975052611627e-05,
      "logits/chosen": -2.081310749053955,
      "logits/rejected": -2.175403118133545,
      "logps/chosen": -210.8900909423828,
      "logps/rejected": -279.8389892578125,
      "loss": 0.0466,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.361638069152832,
      "rewards/margins": 7.6152801513671875,
      "rewards/rejected": -12.976919174194336,
      "step": 1967
    },
    {
      "epoch": 2.58,
      "learning_rate": 4.280243890504419e-05,
      "logits/chosen": -2.237908363342285,
      "logits/rejected": -2.206061601638794,
      "logps/chosen": -254.23226928710938,
      "logps/rejected": -341.8685302734375,
      "loss": 0.0449,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.933377742767334,
      "rewards/margins": 8.773107528686523,
      "rewards/rejected": -13.706486701965332,
      "step": 1968
    },
    {
      "epoch": 2.58,
      "learning_rate": 4.279512419341657e-05,
      "logits/chosen": -2.1924283504486084,
      "logits/rejected": -2.058635950088501,
      "logps/chosen": -260.7196350097656,
      "logps/rejected": -359.5116271972656,
      "loss": 0.0917,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.284665107727051,
      "rewards/margins": 7.44911527633667,
      "rewards/rejected": -12.733780860900879,
      "step": 1969
    },
    {
      "epoch": 2.58,
      "learning_rate": 4.278780639250328e-05,
      "logits/chosen": -2.099184274673462,
      "logits/rejected": -2.174762010574341,
      "logps/chosen": -282.62225341796875,
      "logps/rejected": -377.4095153808594,
      "loss": 0.1133,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.72321081161499,
      "rewards/margins": 6.761861801147461,
      "rewards/rejected": -12.48507308959961,
      "step": 1970
    },
    {
      "epoch": 2.58,
      "learning_rate": 4.27804855035747e-05,
      "logits/chosen": -1.8911799192428589,
      "logits/rejected": -2.0045738220214844,
      "logps/chosen": -269.79632568359375,
      "logps/rejected": -364.15740966796875,
      "loss": 0.0892,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.91278076171875,
      "rewards/margins": 7.7511305809021,
      "rewards/rejected": -12.663909912109375,
      "step": 1971
    },
    {
      "epoch": 2.58,
      "learning_rate": 4.277316152790177e-05,
      "logits/chosen": -1.9748018980026245,
      "logits/rejected": -2.0676732063293457,
      "logps/chosen": -235.08203125,
      "logps/rejected": -371.6739807128906,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.827308177947998,
      "rewards/margins": 9.296005249023438,
      "rewards/rejected": -15.123311996459961,
      "step": 1972
    },
    {
      "epoch": 2.58,
      "learning_rate": 4.276583446675595e-05,
      "logits/chosen": -2.147160291671753,
      "logits/rejected": -2.117455005645752,
      "logps/chosen": -238.53048706054688,
      "logps/rejected": -305.900390625,
      "loss": 0.0106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.343995571136475,
      "rewards/margins": 7.646453857421875,
      "rewards/rejected": -12.990449905395508,
      "step": 1973
    },
    {
      "epoch": 2.58,
      "learning_rate": 4.275850432140923e-05,
      "logits/chosen": -2.092256546020508,
      "logits/rejected": -2.1781668663024902,
      "logps/chosen": -225.18380737304688,
      "logps/rejected": -339.3202819824219,
      "loss": 0.0457,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.446767807006836,
      "rewards/margins": 8.153047561645508,
      "rewards/rejected": -13.599814414978027,
      "step": 1974
    },
    {
      "epoch": 2.58,
      "learning_rate": 4.275117109313415e-05,
      "logits/chosen": -1.8921672105789185,
      "logits/rejected": -1.9029520750045776,
      "logps/chosen": -227.0023651123047,
      "logps/rejected": -343.0833740234375,
      "loss": 0.1068,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.755160331726074,
      "rewards/margins": 7.769000053405762,
      "rewards/rejected": -12.52415943145752,
      "step": 1975
    },
    {
      "epoch": 2.59,
      "learning_rate": 4.2743834783203785e-05,
      "logits/chosen": -2.2617921829223633,
      "logits/rejected": -2.29345440864563,
      "logps/chosen": -250.0797119140625,
      "logps/rejected": -349.07073974609375,
      "loss": 0.0253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.5307135581970215,
      "rewards/margins": 8.098857879638672,
      "rewards/rejected": -12.629570007324219,
      "step": 1976
    },
    {
      "epoch": 2.59,
      "learning_rate": 4.273649539289174e-05,
      "logits/chosen": -2.126615524291992,
      "logits/rejected": -2.1196091175079346,
      "logps/chosen": -215.05816650390625,
      "logps/rejected": -294.6878662109375,
      "loss": 0.0083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.706331253051758,
      "rewards/margins": 9.184209823608398,
      "rewards/rejected": -13.890541076660156,
      "step": 1977
    },
    {
      "epoch": 2.59,
      "learning_rate": 4.272915292347215e-05,
      "logits/chosen": -2.0412535667419434,
      "logits/rejected": -2.0456371307373047,
      "logps/chosen": -299.03790283203125,
      "logps/rejected": -320.651123046875,
      "loss": 0.0272,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.878506660461426,
      "rewards/margins": 7.792044162750244,
      "rewards/rejected": -12.670549392700195,
      "step": 1978
    },
    {
      "epoch": 2.59,
      "learning_rate": 4.272180737621968e-05,
      "logits/chosen": -2.234879493713379,
      "logits/rejected": -2.155164957046509,
      "logps/chosen": -241.6581268310547,
      "logps/rejected": -274.40850830078125,
      "loss": 0.128,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.20209264755249,
      "rewards/margins": 7.107535362243652,
      "rewards/rejected": -12.3096284866333,
      "step": 1979
    },
    {
      "epoch": 2.59,
      "learning_rate": 4.271445875240956e-05,
      "logits/chosen": -2.014817714691162,
      "logits/rejected": -2.011441469192505,
      "logps/chosen": -248.09115600585938,
      "logps/rejected": -301.4862365722656,
      "loss": 0.054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.681390762329102,
      "rewards/margins": 7.81306791305542,
      "rewards/rejected": -12.494458198547363,
      "step": 1980
    },
    {
      "epoch": 2.59,
      "learning_rate": 4.270710705331752e-05,
      "logits/chosen": -2.0360469818115234,
      "logits/rejected": -2.0951154232025146,
      "logps/chosen": -235.5208740234375,
      "logps/rejected": -318.1795959472656,
      "loss": 0.0456,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.8345842361450195,
      "rewards/margins": 8.38646411895752,
      "rewards/rejected": -13.221047401428223,
      "step": 1981
    },
    {
      "epoch": 2.59,
      "learning_rate": 4.269975228021983e-05,
      "logits/chosen": -2.0260274410247803,
      "logits/rejected": -2.068878412246704,
      "logps/chosen": -277.013671875,
      "logps/rejected": -365.9017639160156,
      "loss": 0.0443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.482072353363037,
      "rewards/margins": 9.523672103881836,
      "rewards/rejected": -14.005743980407715,
      "step": 1982
    },
    {
      "epoch": 2.6,
      "learning_rate": 4.2692394434393313e-05,
      "logits/chosen": -2.2093918323516846,
      "logits/rejected": -2.239203453063965,
      "logps/chosen": -237.828125,
      "logps/rejected": -312.05755615234375,
      "loss": 0.1387,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.4257378578186035,
      "rewards/margins": 7.293093681335449,
      "rewards/rejected": -11.718832969665527,
      "step": 1983
    },
    {
      "epoch": 2.6,
      "learning_rate": 4.268503351711531e-05,
      "logits/chosen": -1.9702582359313965,
      "logits/rejected": -2.1271867752075195,
      "logps/chosen": -304.22900390625,
      "logps/rejected": -398.12481689453125,
      "loss": 0.0886,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.615696907043457,
      "rewards/margins": 7.297685623168945,
      "rewards/rejected": -13.913382530212402,
      "step": 1984
    },
    {
      "epoch": 2.6,
      "learning_rate": 4.267766952966369e-05,
      "logits/chosen": -2.177107810974121,
      "logits/rejected": -2.2635247707366943,
      "logps/chosen": -201.6862030029297,
      "logps/rejected": -304.0697021484375,
      "loss": 0.0172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.194709777832031,
      "rewards/margins": 8.228707313537598,
      "rewards/rejected": -12.423417091369629,
      "step": 1985
    },
    {
      "epoch": 2.6,
      "learning_rate": 4.267030247331687e-05,
      "logits/chosen": -1.8618240356445312,
      "logits/rejected": -1.7992628812789917,
      "logps/chosen": -226.5166015625,
      "logps/rejected": -268.01641845703125,
      "loss": 0.0919,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.920095920562744,
      "rewards/margins": 7.572173118591309,
      "rewards/rejected": -11.492269515991211,
      "step": 1986
    },
    {
      "epoch": 2.6,
      "learning_rate": 4.266293234935379e-05,
      "logits/chosen": -2.151167392730713,
      "logits/rejected": -2.200986385345459,
      "logps/chosen": -209.70037841796875,
      "logps/rejected": -284.14422607421875,
      "loss": 0.0521,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.9986391067504883,
      "rewards/margins": 6.333474159240723,
      "rewards/rejected": -10.332113265991211,
      "step": 1987
    },
    {
      "epoch": 2.6,
      "learning_rate": 4.265555915905393e-05,
      "logits/chosen": -1.9930312633514404,
      "logits/rejected": -1.8740456104278564,
      "logps/chosen": -240.62808227539062,
      "logps/rejected": -283.4354553222656,
      "loss": 0.1025,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.969852447509766,
      "rewards/margins": 6.814610004425049,
      "rewards/rejected": -11.784461975097656,
      "step": 1988
    },
    {
      "epoch": 2.6,
      "learning_rate": 4.264818290369729e-05,
      "logits/chosen": -2.1442923545837402,
      "logits/rejected": -2.1885173320770264,
      "logps/chosen": -227.04296875,
      "logps/rejected": -342.5315856933594,
      "loss": 0.0049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.07236909866333,
      "rewards/margins": 9.052876472473145,
      "rewards/rejected": -14.125244140625,
      "step": 1989
    },
    {
      "epoch": 2.6,
      "learning_rate": 4.264080358456441e-05,
      "logits/chosen": -1.810939908027649,
      "logits/rejected": -1.8494899272918701,
      "logps/chosen": -250.21864318847656,
      "logps/rejected": -322.9114990234375,
      "loss": 0.0893,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.772063255310059,
      "rewards/margins": 7.34108829498291,
      "rewards/rejected": -12.113152503967285,
      "step": 1990
    },
    {
      "epoch": 2.61,
      "learning_rate": 4.263342120293636e-05,
      "logits/chosen": -1.9354889392852783,
      "logits/rejected": -2.0048763751983643,
      "logps/chosen": -231.1033477783203,
      "logps/rejected": -296.833984375,
      "loss": 0.0519,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.519188404083252,
      "rewards/margins": 8.479740142822266,
      "rewards/rejected": -11.998929023742676,
      "step": 1991
    },
    {
      "epoch": 2.61,
      "learning_rate": 4.2626035760094763e-05,
      "logits/chosen": -2.1611340045928955,
      "logits/rejected": -2.1724023818969727,
      "logps/chosen": -274.818359375,
      "logps/rejected": -365.2533874511719,
      "loss": 0.044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.13316535949707,
      "rewards/margins": 7.889025688171387,
      "rewards/rejected": -12.02219009399414,
      "step": 1992
    },
    {
      "epoch": 2.61,
      "learning_rate": 4.261864725732172e-05,
      "logits/chosen": -2.252372980117798,
      "logits/rejected": -2.216064453125,
      "logps/chosen": -274.22711181640625,
      "logps/rejected": -386.2915344238281,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.8807830810546875,
      "rewards/margins": 9.966548919677734,
      "rewards/rejected": -13.847332954406738,
      "step": 1993
    },
    {
      "epoch": 2.61,
      "learning_rate": 4.261125569589993e-05,
      "logits/chosen": -2.0184173583984375,
      "logits/rejected": -2.0977327823638916,
      "logps/chosen": -200.14089965820312,
      "logps/rejected": -281.9545593261719,
      "loss": 0.028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.331934928894043,
      "rewards/margins": 8.05714225769043,
      "rewards/rejected": -11.389076232910156,
      "step": 1994
    },
    {
      "epoch": 2.61,
      "learning_rate": 4.260386107711256e-05,
      "logits/chosen": -1.8014366626739502,
      "logits/rejected": -1.7428510189056396,
      "logps/chosen": -237.07882690429688,
      "logps/rejected": -313.3893737792969,
      "loss": 0.0968,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.013710021972656,
      "rewards/margins": 6.934698581695557,
      "rewards/rejected": -10.948408126831055,
      "step": 1995
    },
    {
      "epoch": 2.61,
      "learning_rate": 4.259646340224336e-05,
      "logits/chosen": -2.0515670776367188,
      "logits/rejected": -2.1084394454956055,
      "logps/chosen": -253.11888122558594,
      "logps/rejected": -400.4759216308594,
      "loss": 0.0211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.113040924072266,
      "rewards/margins": 8.693325996398926,
      "rewards/rejected": -13.806365966796875,
      "step": 1996
    },
    {
      "epoch": 2.61,
      "learning_rate": 4.2589062672576576e-05,
      "logits/chosen": -1.991852879524231,
      "logits/rejected": -2.1616296768188477,
      "logps/chosen": -201.46896362304688,
      "logps/rejected": -320.1032409667969,
      "loss": 0.0021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.442726135253906,
      "rewards/margins": 8.485641479492188,
      "rewards/rejected": -12.928366661071777,
      "step": 1997
    },
    {
      "epoch": 2.61,
      "learning_rate": 4.2581658889397e-05,
      "logits/chosen": -1.7517123222351074,
      "logits/rejected": -1.7525990009307861,
      "logps/chosen": -241.78564453125,
      "logps/rejected": -282.4005126953125,
      "loss": 0.1314,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.3433334827423096,
      "rewards/margins": 7.258170127868652,
      "rewards/rejected": -10.6015043258667,
      "step": 1998
    },
    {
      "epoch": 2.62,
      "learning_rate": 4.257425205398995e-05,
      "logits/chosen": -2.0399134159088135,
      "logits/rejected": -2.096323013305664,
      "logps/chosen": -213.169677734375,
      "logps/rejected": -356.84088134765625,
      "loss": 0.1302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.694550037384033,
      "rewards/margins": 9.911086082458496,
      "rewards/rejected": -13.605636596679688,
      "step": 1999
    },
    {
      "epoch": 2.62,
      "learning_rate": 4.256684216764127e-05,
      "logits/chosen": -2.1457009315490723,
      "logits/rejected": -2.224804401397705,
      "logps/chosen": -271.61932373046875,
      "logps/rejected": -375.3747863769531,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.7626380920410156,
      "rewards/margins": 9.301042556762695,
      "rewards/rejected": -13.063680648803711,
      "step": 2000
    },
    {
      "epoch": 2.62,
      "learning_rate": 4.255942923163736e-05,
      "logits/chosen": -2.1172068119049072,
      "logits/rejected": -2.1737592220306396,
      "logps/chosen": -203.1175537109375,
      "logps/rejected": -280.4482727050781,
      "loss": 0.1161,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.822445869445801,
      "rewards/margins": 6.7819695472717285,
      "rewards/rejected": -11.604415893554688,
      "step": 2001
    },
    {
      "epoch": 2.62,
      "learning_rate": 4.2552013247265096e-05,
      "logits/chosen": -1.992425560951233,
      "logits/rejected": -1.9609733819961548,
      "logps/chosen": -237.23434448242188,
      "logps/rejected": -298.7255554199219,
      "loss": 0.0156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.9892876148223877,
      "rewards/margins": 7.11054801940918,
      "rewards/rejected": -11.099835395812988,
      "step": 2002
    },
    {
      "epoch": 2.62,
      "learning_rate": 4.2544594215811936e-05,
      "logits/chosen": -2.161130428314209,
      "logits/rejected": -2.1829681396484375,
      "logps/chosen": -256.2269287109375,
      "logps/rejected": -337.739013671875,
      "loss": 0.09,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.339508533477783,
      "rewards/margins": 6.66135311126709,
      "rewards/rejected": -11.000860214233398,
      "step": 2003
    },
    {
      "epoch": 2.62,
      "learning_rate": 4.253717213856583e-05,
      "logits/chosen": -1.951271891593933,
      "logits/rejected": -2.142191171646118,
      "logps/chosen": -194.9743194580078,
      "logps/rejected": -322.3970031738281,
      "loss": 0.0223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.501600742340088,
      "rewards/margins": 9.656272888183594,
      "rewards/rejected": -14.157873153686523,
      "step": 2004
    },
    {
      "epoch": 2.62,
      "learning_rate": 4.252974701681528e-05,
      "logits/chosen": -2.2527217864990234,
      "logits/rejected": -2.328521728515625,
      "logps/chosen": -218.23036193847656,
      "logps/rejected": -293.6946105957031,
      "loss": 0.046,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.480393886566162,
      "rewards/margins": 6.972497940063477,
      "rewards/rejected": -11.452892303466797,
      "step": 2005
    },
    {
      "epoch": 2.63,
      "learning_rate": 4.252231885184932e-05,
      "logits/chosen": -2.1791348457336426,
      "logits/rejected": -2.2902534008026123,
      "logps/chosen": -266.1272888183594,
      "logps/rejected": -347.8664855957031,
      "loss": 0.0595,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.511211395263672,
      "rewards/margins": 6.929591655731201,
      "rewards/rejected": -12.440802574157715,
      "step": 2006
    },
    {
      "epoch": 2.63,
      "learning_rate": 4.251488764495748e-05,
      "logits/chosen": -1.8994340896606445,
      "logits/rejected": -1.9029484987258911,
      "logps/chosen": -256.0934143066406,
      "logps/rejected": -326.8026428222656,
      "loss": 0.0878,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.944005012512207,
      "rewards/margins": 7.92893648147583,
      "rewards/rejected": -12.872941017150879,
      "step": 2007
    },
    {
      "epoch": 2.63,
      "learning_rate": 4.2507453397429855e-05,
      "logits/chosen": -1.8833651542663574,
      "logits/rejected": -2.036100149154663,
      "logps/chosen": -243.87295532226562,
      "logps/rejected": -350.85479736328125,
      "loss": 0.0291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.893523693084717,
      "rewards/margins": 7.373872756958008,
      "rewards/rejected": -12.267395973205566,
      "step": 2008
    },
    {
      "epoch": 2.63,
      "learning_rate": 4.2500016110557044e-05,
      "logits/chosen": -2.0685205459594727,
      "logits/rejected": -2.0962882041931152,
      "logps/chosen": -209.53982543945312,
      "logps/rejected": -284.69525146484375,
      "loss": 0.0324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.660156726837158,
      "rewards/margins": 7.2062087059021,
      "rewards/rejected": -11.866365432739258,
      "step": 2009
    },
    {
      "epoch": 2.63,
      "learning_rate": 4.249257578563019e-05,
      "logits/chosen": -2.0176610946655273,
      "logits/rejected": -2.0090808868408203,
      "logps/chosen": -218.67474365234375,
      "logps/rejected": -309.4306945800781,
      "loss": 0.0883,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.934572219848633,
      "rewards/margins": 10.24348258972168,
      "rewards/rejected": -13.178053855895996,
      "step": 2010
    },
    {
      "epoch": 2.63,
      "learning_rate": 4.2485132423940944e-05,
      "logits/chosen": -2.4200572967529297,
      "logits/rejected": -2.2854561805725098,
      "logps/chosen": -251.91578674316406,
      "logps/rejected": -281.0067138671875,
      "loss": 0.1013,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.3066606521606445,
      "rewards/margins": 5.664590358734131,
      "rewards/rejected": -10.971250534057617,
      "step": 2011
    },
    {
      "epoch": 2.63,
      "learning_rate": 4.2477686026781505e-05,
      "logits/chosen": -1.8213557004928589,
      "logits/rejected": -1.8547894954681396,
      "logps/chosen": -276.2442932128906,
      "logps/rejected": -335.45361328125,
      "loss": 0.1162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.994606971740723,
      "rewards/margins": 7.177623271942139,
      "rewards/rejected": -13.17223072052002,
      "step": 2012
    },
    {
      "epoch": 2.63,
      "learning_rate": 4.247023659544458e-05,
      "logits/chosen": -1.8445768356323242,
      "logits/rejected": -1.8069664239883423,
      "logps/chosen": -237.87130737304688,
      "logps/rejected": -332.58685302734375,
      "loss": 0.1008,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.717880725860596,
      "rewards/margins": 7.067141056060791,
      "rewards/rejected": -12.785021781921387,
      "step": 2013
    },
    {
      "epoch": 2.64,
      "learning_rate": 4.2462784131223434e-05,
      "logits/chosen": -1.8459657430648804,
      "logits/rejected": -1.9359010457992554,
      "logps/chosen": -231.81597900390625,
      "logps/rejected": -315.25128173828125,
      "loss": 0.0558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.855999231338501,
      "rewards/margins": 8.611515045166016,
      "rewards/rejected": -12.467513084411621,
      "step": 2014
    },
    {
      "epoch": 2.64,
      "learning_rate": 4.24553286354118e-05,
      "logits/chosen": -1.9260315895080566,
      "logits/rejected": -2.0133044719696045,
      "logps/chosen": -242.34796142578125,
      "logps/rejected": -330.73486328125,
      "loss": 0.0883,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.197949409484863,
      "rewards/margins": 7.258923530578613,
      "rewards/rejected": -12.456872940063477,
      "step": 2015
    },
    {
      "epoch": 2.64,
      "learning_rate": 4.2447870109304007e-05,
      "logits/chosen": -2.1096689701080322,
      "logits/rejected": -2.100594997406006,
      "logps/chosen": -176.09457397460938,
      "logps/rejected": -262.0912780761719,
      "loss": 0.0945,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.2592055797576904,
      "rewards/margins": 6.781350135803223,
      "rewards/rejected": -10.040555000305176,
      "step": 2016
    },
    {
      "epoch": 2.64,
      "learning_rate": 4.244040855419486e-05,
      "logits/chosen": -2.171529769897461,
      "logits/rejected": -2.1618311405181885,
      "logps/chosen": -303.1656494140625,
      "logps/rejected": -379.5803527832031,
      "loss": 0.0464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.887218952178955,
      "rewards/margins": 8.025300025939941,
      "rewards/rejected": -12.912520408630371,
      "step": 2017
    },
    {
      "epoch": 2.64,
      "learning_rate": 4.243294397137971e-05,
      "logits/chosen": -1.879332423210144,
      "logits/rejected": -1.950081467628479,
      "logps/chosen": -233.27285766601562,
      "logps/rejected": -300.0467529296875,
      "loss": 0.0454,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.6014180183410645,
      "rewards/margins": 7.412932395935059,
      "rewards/rejected": -12.014349937438965,
      "step": 2018
    },
    {
      "epoch": 2.64,
      "learning_rate": 4.2425476362154436e-05,
      "logits/chosen": -2.1640138626098633,
      "logits/rejected": -2.1587491035461426,
      "logps/chosen": -239.59934997558594,
      "logps/rejected": -304.4157409667969,
      "loss": 0.0461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.2458882331848145,
      "rewards/margins": 7.059645652770996,
      "rewards/rejected": -12.305534362792969,
      "step": 2019
    },
    {
      "epoch": 2.64,
      "learning_rate": 4.241800572781543e-05,
      "logits/chosen": -2.160501718521118,
      "logits/rejected": -2.304579496383667,
      "logps/chosen": -261.8168029785156,
      "logps/rejected": -395.4276428222656,
      "loss": 0.0165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.653003215789795,
      "rewards/margins": 9.462180137634277,
      "rewards/rejected": -15.115182876586914,
      "step": 2020
    },
    {
      "epoch": 2.64,
      "learning_rate": 4.241053206965962e-05,
      "logits/chosen": -1.970011830329895,
      "logits/rejected": -2.1001381874084473,
      "logps/chosen": -247.37672424316406,
      "logps/rejected": -360.9378662109375,
      "loss": 0.0853,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.5289154052734375,
      "rewards/margins": 7.392343521118164,
      "rewards/rejected": -12.921258926391602,
      "step": 2021
    },
    {
      "epoch": 2.65,
      "learning_rate": 4.240305538898445e-05,
      "logits/chosen": -2.243610143661499,
      "logits/rejected": -2.2371904850006104,
      "logps/chosen": -259.6739807128906,
      "logps/rejected": -319.3279113769531,
      "loss": 0.1314,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.689486026763916,
      "rewards/margins": 7.2585906982421875,
      "rewards/rejected": -11.948076248168945,
      "step": 2022
    },
    {
      "epoch": 2.65,
      "learning_rate": 4.23955756870879e-05,
      "logits/chosen": -2.2176706790924072,
      "logits/rejected": -2.1681039333343506,
      "logps/chosen": -203.82867431640625,
      "logps/rejected": -276.8878479003906,
      "loss": 0.0925,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.021929740905762,
      "rewards/margins": 6.763903617858887,
      "rewards/rejected": -11.785833358764648,
      "step": 2023
    },
    {
      "epoch": 2.65,
      "learning_rate": 4.238809296526847e-05,
      "logits/chosen": -1.669848918914795,
      "logits/rejected": -1.7608784437179565,
      "logps/chosen": -234.96856689453125,
      "logps/rejected": -347.46417236328125,
      "loss": 0.0494,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.062384128570557,
      "rewards/margins": 8.291193962097168,
      "rewards/rejected": -13.353578567504883,
      "step": 2024
    },
    {
      "epoch": 2.65,
      "learning_rate": 4.238060722482517e-05,
      "logits/chosen": -1.968036413192749,
      "logits/rejected": -2.05326509475708,
      "logps/chosen": -270.92071533203125,
      "logps/rejected": -378.5557861328125,
      "loss": 0.0983,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.9035468101501465,
      "rewards/margins": 7.38877534866333,
      "rewards/rejected": -13.292322158813477,
      "step": 2025
    },
    {
      "epoch": 2.65,
      "learning_rate": 4.237311846705755e-05,
      "logits/chosen": -2.2504067420959473,
      "logits/rejected": -2.1848998069763184,
      "logps/chosen": -256.95086669921875,
      "logps/rejected": -323.0563049316406,
      "loss": 0.0443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.150045394897461,
      "rewards/margins": 9.079219818115234,
      "rewards/rejected": -14.229267120361328,
      "step": 2026
    },
    {
      "epoch": 2.65,
      "learning_rate": 4.2365626693265694e-05,
      "logits/chosen": -1.9853413105010986,
      "logits/rejected": -1.9951058626174927,
      "logps/chosen": -275.75726318359375,
      "logps/rejected": -333.6006164550781,
      "loss": 0.0894,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.940235614776611,
      "rewards/margins": 8.546769142150879,
      "rewards/rejected": -13.487004280090332,
      "step": 2027
    },
    {
      "epoch": 2.65,
      "learning_rate": 4.235813190475018e-05,
      "logits/chosen": -2.1142847537994385,
      "logits/rejected": -2.0551466941833496,
      "logps/chosen": -197.76902770996094,
      "logps/rejected": -251.27593994140625,
      "loss": 0.0499,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.9538817405700684,
      "rewards/margins": 8.75600528717041,
      "rewards/rejected": -12.70988655090332,
      "step": 2028
    },
    {
      "epoch": 2.66,
      "learning_rate": 4.2350634102812134e-05,
      "logits/chosen": -1.9945062398910522,
      "logits/rejected": -2.0655226707458496,
      "logps/chosen": -203.58526611328125,
      "logps/rejected": -327.26776123046875,
      "loss": 0.0887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.519442558288574,
      "rewards/margins": 6.988471984863281,
      "rewards/rejected": -11.507914543151855,
      "step": 2029
    },
    {
      "epoch": 2.66,
      "learning_rate": 4.2343133288753194e-05,
      "logits/chosen": -1.9564071893692017,
      "logits/rejected": -2.0216352939605713,
      "logps/chosen": -222.12017822265625,
      "logps/rejected": -285.66717529296875,
      "loss": 0.0892,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.716222763061523,
      "rewards/margins": 7.841244220733643,
      "rewards/rejected": -12.557466506958008,
      "step": 2030
    },
    {
      "epoch": 2.66,
      "learning_rate": 4.233562946387553e-05,
      "logits/chosen": -1.8869245052337646,
      "logits/rejected": -1.865543246269226,
      "logps/chosen": -232.640380859375,
      "logps/rejected": -314.2353820800781,
      "loss": 0.0494,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.838282108306885,
      "rewards/margins": 7.584604740142822,
      "rewards/rejected": -12.422886848449707,
      "step": 2031
    },
    {
      "epoch": 2.66,
      "learning_rate": 4.232812262948182e-05,
      "logits/chosen": -1.9131499528884888,
      "logits/rejected": -2.000537395477295,
      "logps/chosen": -198.13389587402344,
      "logps/rejected": -307.91473388671875,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.581729888916016,
      "rewards/margins": 8.919013023376465,
      "rewards/rejected": -13.50074291229248,
      "step": 2032
    },
    {
      "epoch": 2.66,
      "learning_rate": 4.232061278687526e-05,
      "logits/chosen": -1.8416846990585327,
      "logits/rejected": -1.8420649766921997,
      "logps/chosen": -271.35919189453125,
      "logps/rejected": -370.6395568847656,
      "loss": 0.0871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.359163284301758,
      "rewards/margins": 8.96020793914795,
      "rewards/rejected": -14.319372177124023,
      "step": 2033
    },
    {
      "epoch": 2.66,
      "learning_rate": 4.23130999373596e-05,
      "logits/chosen": -2.199634075164795,
      "logits/rejected": -2.315343141555786,
      "logps/chosen": -244.52755737304688,
      "logps/rejected": -343.6764831542969,
      "loss": 0.0091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.335355281829834,
      "rewards/margins": 9.197492599487305,
      "rewards/rejected": -13.532848358154297,
      "step": 2034
    },
    {
      "epoch": 2.66,
      "learning_rate": 4.230558408223909e-05,
      "logits/chosen": -2.3857579231262207,
      "logits/rejected": -2.346330404281616,
      "logps/chosen": -286.7845153808594,
      "logps/rejected": -358.5130310058594,
      "loss": 0.0875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.38930606842041,
      "rewards/margins": 7.837110996246338,
      "rewards/rejected": -13.22641658782959,
      "step": 2035
    },
    {
      "epoch": 2.66,
      "learning_rate": 4.22980652228185e-05,
      "logits/chosen": -1.974481463432312,
      "logits/rejected": -1.9705196619033813,
      "logps/chosen": -259.6114807128906,
      "logps/rejected": -350.9654235839844,
      "loss": 0.1726,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.30286169052124,
      "rewards/margins": 8.534625053405762,
      "rewards/rejected": -13.837486267089844,
      "step": 2036
    },
    {
      "epoch": 2.67,
      "learning_rate": 4.229054336040312e-05,
      "logits/chosen": -1.9328861236572266,
      "logits/rejected": -1.965613842010498,
      "logps/chosen": -223.0769500732422,
      "logps/rejected": -322.9892578125,
      "loss": 0.0453,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.8855221271514893,
      "rewards/margins": 10.301925659179688,
      "rewards/rejected": -14.187447547912598,
      "step": 2037
    },
    {
      "epoch": 2.67,
      "learning_rate": 4.228301849629879e-05,
      "logits/chosen": -2.1937811374664307,
      "logits/rejected": -2.216660737991333,
      "logps/chosen": -253.26025390625,
      "logps/rejected": -344.1639099121094,
      "loss": 0.1048,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.741037368774414,
      "rewards/margins": 8.45821762084961,
      "rewards/rejected": -13.19925594329834,
      "step": 2038
    },
    {
      "epoch": 2.67,
      "learning_rate": 4.227549063181183e-05,
      "logits/chosen": -2.151825189590454,
      "logits/rejected": -2.158660650253296,
      "logps/chosen": -276.98419189453125,
      "logps/rejected": -352.35699462890625,
      "loss": 0.0375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.893413543701172,
      "rewards/margins": 9.326420783996582,
      "rewards/rejected": -14.219833374023438,
      "step": 2039
    },
    {
      "epoch": 2.67,
      "learning_rate": 4.22679597682491e-05,
      "logits/chosen": -2.0169548988342285,
      "logits/rejected": -2.21177077293396,
      "logps/chosen": -262.75469970703125,
      "logps/rejected": -369.38201904296875,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.374359607696533,
      "rewards/margins": 8.58126449584961,
      "rewards/rejected": -13.955623626708984,
      "step": 2040
    },
    {
      "epoch": 2.67,
      "learning_rate": 4.2260425906917975e-05,
      "logits/chosen": -2.1499552726745605,
      "logits/rejected": -2.195781946182251,
      "logps/chosen": -215.60107421875,
      "logps/rejected": -310.01287841796875,
      "loss": 0.133,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.024834156036377,
      "rewards/margins": 7.125798225402832,
      "rewards/rejected": -11.15063190460205,
      "step": 2041
    },
    {
      "epoch": 2.67,
      "learning_rate": 4.2252889049126366e-05,
      "logits/chosen": -2.129696846008301,
      "logits/rejected": -2.079493284225464,
      "logps/chosen": -232.66456604003906,
      "logps/rejected": -298.35205078125,
      "loss": 0.094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.621220588684082,
      "rewards/margins": 7.01010799407959,
      "rewards/rejected": -11.631328582763672,
      "step": 2042
    },
    {
      "epoch": 2.67,
      "learning_rate": 4.224534919618269e-05,
      "logits/chosen": -2.1040329933166504,
      "logits/rejected": -2.150583267211914,
      "logps/chosen": -217.6129608154297,
      "logps/rejected": -315.8685302734375,
      "loss": 0.0523,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.8543291091918945,
      "rewards/margins": 8.18506908416748,
      "rewards/rejected": -13.039399147033691,
      "step": 2043
    },
    {
      "epoch": 2.67,
      "learning_rate": 4.22378063493959e-05,
      "logits/chosen": -2.154564380645752,
      "logits/rejected": -2.1789019107818604,
      "logps/chosen": -208.6300506591797,
      "logps/rejected": -282.248046875,
      "loss": 0.04,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.85252046585083,
      "rewards/margins": 7.109794616699219,
      "rewards/rejected": -10.96231460571289,
      "step": 2044
    },
    {
      "epoch": 2.68,
      "learning_rate": 4.223026051007544e-05,
      "logits/chosen": -1.9202934503555298,
      "logits/rejected": -1.9896492958068848,
      "logps/chosen": -287.2677001953125,
      "logps/rejected": -405.3075256347656,
      "loss": 0.0458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.884298324584961,
      "rewards/margins": 8.85289478302002,
      "rewards/rejected": -12.73719310760498,
      "step": 2045
    },
    {
      "epoch": 2.68,
      "learning_rate": 4.222271167953128e-05,
      "logits/chosen": -1.9422993659973145,
      "logits/rejected": -1.962449073791504,
      "logps/chosen": -291.1929626464844,
      "logps/rejected": -366.3737487792969,
      "loss": 0.0457,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.793270111083984,
      "rewards/margins": 7.609700679779053,
      "rewards/rejected": -12.402970314025879,
      "step": 2046
    },
    {
      "epoch": 2.68,
      "learning_rate": 4.2215159859073944e-05,
      "logits/chosen": -2.332904577255249,
      "logits/rejected": -2.3232131004333496,
      "logps/chosen": -207.947998046875,
      "logps/rejected": -289.6778564453125,
      "loss": 0.0938,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.780367374420166,
      "rewards/margins": 8.799994468688965,
      "rewards/rejected": -12.580362319946289,
      "step": 2047
    },
    {
      "epoch": 2.68,
      "learning_rate": 4.2207605050014435e-05,
      "logits/chosen": -2.3431074619293213,
      "logits/rejected": -2.3058929443359375,
      "logps/chosen": -224.7828369140625,
      "logps/rejected": -314.53668212890625,
      "loss": 0.1395,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.5304112434387207,
      "rewards/margins": 6.5715413093566895,
      "rewards/rejected": -10.10195255279541,
      "step": 2048
    },
    {
      "epoch": 2.68,
      "learning_rate": 4.22000472536643e-05,
      "logits/chosen": -1.9623531103134155,
      "logits/rejected": -2.0505189895629883,
      "logps/chosen": -237.1623077392578,
      "logps/rejected": -324.4232177734375,
      "loss": 0.0067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.392932415008545,
      "rewards/margins": 7.678095817565918,
      "rewards/rejected": -12.071028709411621,
      "step": 2049
    },
    {
      "epoch": 2.68,
      "learning_rate": 4.2192486471335585e-05,
      "logits/chosen": -2.3527064323425293,
      "logits/rejected": -2.3956918716430664,
      "logps/chosen": -237.3239288330078,
      "logps/rejected": -343.18817138671875,
      "loss": 0.0481,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.818669557571411,
      "rewards/margins": 9.966001510620117,
      "rewards/rejected": -13.784668922424316,
      "step": 2050
    },
    {
      "epoch": 2.68,
      "learning_rate": 4.2184922704340866e-05,
      "logits/chosen": -2.1499099731445312,
      "logits/rejected": -2.116560935974121,
      "logps/chosen": -213.3365936279297,
      "logps/rejected": -318.2062072753906,
      "loss": 0.0703,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.0955376625061035,
      "rewards/margins": 8.705997467041016,
      "rewards/rejected": -12.801535606384277,
      "step": 2051
    },
    {
      "epoch": 2.69,
      "learning_rate": 4.217735595399324e-05,
      "logits/chosen": -2.3184332847595215,
      "logits/rejected": -2.389354944229126,
      "logps/chosen": -230.14830017089844,
      "logps/rejected": -325.69573974609375,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.083515167236328,
      "rewards/margins": 9.438323020935059,
      "rewards/rejected": -13.52183723449707,
      "step": 2052
    },
    {
      "epoch": 2.69,
      "learning_rate": 4.2169786221606315e-05,
      "logits/chosen": -2.1536691188812256,
      "logits/rejected": -2.2255914211273193,
      "logps/chosen": -208.54898071289062,
      "logps/rejected": -316.613037109375,
      "loss": 0.0439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.340245723724365,
      "rewards/margins": 9.141230583190918,
      "rewards/rejected": -13.481475830078125,
      "step": 2053
    },
    {
      "epoch": 2.69,
      "learning_rate": 4.216221350849422e-05,
      "logits/chosen": -2.342555046081543,
      "logits/rejected": -2.3749189376831055,
      "logps/chosen": -271.072509765625,
      "logps/rejected": -318.45648193359375,
      "loss": 0.0783,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.5725584030151367,
      "rewards/margins": 8.381831169128418,
      "rewards/rejected": -11.954388618469238,
      "step": 2054
    },
    {
      "epoch": 2.69,
      "learning_rate": 4.215463781597159e-05,
      "logits/chosen": -2.0869643688201904,
      "logits/rejected": -2.116182327270508,
      "logps/chosen": -261.493896484375,
      "logps/rejected": -365.8189697265625,
      "loss": 0.0987,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.4405364990234375,
      "rewards/margins": 7.905498504638672,
      "rewards/rejected": -12.346035957336426,
      "step": 2055
    },
    {
      "epoch": 2.69,
      "learning_rate": 4.214705914535361e-05,
      "logits/chosen": -2.2096025943756104,
      "logits/rejected": -2.250379800796509,
      "logps/chosen": -270.6283264160156,
      "logps/rejected": -394.0135803222656,
      "loss": 0.0497,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.837217807769775,
      "rewards/margins": 8.825190544128418,
      "rewards/rejected": -13.662408828735352,
      "step": 2056
    },
    {
      "epoch": 2.69,
      "learning_rate": 4.213947749795593e-05,
      "logits/chosen": -2.2673234939575195,
      "logits/rejected": -2.196312665939331,
      "logps/chosen": -227.28076171875,
      "logps/rejected": -277.54034423828125,
      "loss": 0.0913,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.908783435821533,
      "rewards/margins": 6.475468158721924,
      "rewards/rejected": -11.38425064086914,
      "step": 2057
    },
    {
      "epoch": 2.69,
      "learning_rate": 4.2131892875094756e-05,
      "logits/chosen": -2.2475051879882812,
      "logits/rejected": -2.249516010284424,
      "logps/chosen": -254.70135498046875,
      "logps/rejected": -320.4151306152344,
      "loss": 0.0451,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.25231409072876,
      "rewards/margins": 6.865804195404053,
      "rewards/rejected": -12.118119239807129,
      "step": 2058
    },
    {
      "epoch": 2.69,
      "learning_rate": 4.212430527808682e-05,
      "logits/chosen": -2.326888084411621,
      "logits/rejected": -2.471912384033203,
      "logps/chosen": -258.8665466308594,
      "logps/rejected": -347.352294921875,
      "loss": 0.0453,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.683292388916016,
      "rewards/margins": 7.736345291137695,
      "rewards/rejected": -12.419637680053711,
      "step": 2059
    },
    {
      "epoch": 2.7,
      "learning_rate": 4.211671470824933e-05,
      "logits/chosen": -2.1365933418273926,
      "logits/rejected": -2.1905436515808105,
      "logps/chosen": -235.756103515625,
      "logps/rejected": -328.03961181640625,
      "loss": 0.1719,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.880552291870117,
      "rewards/margins": 7.123145580291748,
      "rewards/rejected": -12.003698348999023,
      "step": 2060
    },
    {
      "epoch": 2.7,
      "learning_rate": 4.210912116690004e-05,
      "logits/chosen": -2.2770068645477295,
      "logits/rejected": -2.2150559425354004,
      "logps/chosen": -282.68084716796875,
      "logps/rejected": -355.66058349609375,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.084074020385742,
      "rewards/margins": 10.210107803344727,
      "rewards/rejected": -14.294180870056152,
      "step": 2061
    },
    {
      "epoch": 2.7,
      "learning_rate": 4.210152465535721e-05,
      "logits/chosen": -1.9398760795593262,
      "logits/rejected": -1.9859808683395386,
      "logps/chosen": -252.51089477539062,
      "logps/rejected": -308.23846435546875,
      "loss": 0.0735,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.215598106384277,
      "rewards/margins": 7.410407543182373,
      "rewards/rejected": -11.626004219055176,
      "step": 2062
    },
    {
      "epoch": 2.7,
      "learning_rate": 4.209392517493961e-05,
      "logits/chosen": -2.078042984008789,
      "logits/rejected": -2.0762107372283936,
      "logps/chosen": -200.397216796875,
      "logps/rejected": -326.5500183105469,
      "loss": 0.0443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.259744644165039,
      "rewards/margins": 8.913934707641602,
      "rewards/rejected": -13.173680305480957,
      "step": 2063
    },
    {
      "epoch": 2.7,
      "learning_rate": 4.2086322726966546e-05,
      "logits/chosen": -2.240522861480713,
      "logits/rejected": -2.254432201385498,
      "logps/chosen": -209.32215881347656,
      "logps/rejected": -310.88641357421875,
      "loss": 0.0067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.953611373901367,
      "rewards/margins": 9.082300186157227,
      "rewards/rejected": -13.035911560058594,
      "step": 2064
    },
    {
      "epoch": 2.7,
      "learning_rate": 4.207871731275781e-05,
      "logits/chosen": -2.1679046154022217,
      "logits/rejected": -2.14817476272583,
      "logps/chosen": -257.8780517578125,
      "logps/rejected": -345.7042541503906,
      "loss": 0.0065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.281774520874023,
      "rewards/margins": 8.699878692626953,
      "rewards/rejected": -12.981653213500977,
      "step": 2065
    },
    {
      "epoch": 2.7,
      "learning_rate": 4.207110893363373e-05,
      "logits/chosen": -2.295118808746338,
      "logits/rejected": -2.2843916416168213,
      "logps/chosen": -238.00244140625,
      "logps/rejected": -338.3034362792969,
      "loss": 0.0455,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.327823162078857,
      "rewards/margins": 8.419173240661621,
      "rewards/rejected": -12.74699592590332,
      "step": 2066
    },
    {
      "epoch": 2.71,
      "learning_rate": 4.206349759091515e-05,
      "logits/chosen": -1.808960199356079,
      "logits/rejected": -1.8792856931686401,
      "logps/chosen": -262.61285400390625,
      "logps/rejected": -356.2742614746094,
      "loss": 0.0874,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.9227986335754395,
      "rewards/margins": 7.593912124633789,
      "rewards/rejected": -13.516711235046387,
      "step": 2067
    },
    {
      "epoch": 2.71,
      "learning_rate": 4.205588328592341e-05,
      "logits/chosen": -2.258066177368164,
      "logits/rejected": -2.3306429386138916,
      "logps/chosen": -232.1697998046875,
      "logps/rejected": -367.2823181152344,
      "loss": 0.0439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.021013259887695,
      "rewards/margins": 10.31048583984375,
      "rewards/rejected": -15.331500053405762,
      "step": 2068
    },
    {
      "epoch": 2.71,
      "learning_rate": 4.2048266019980385e-05,
      "logits/chosen": -2.03212833404541,
      "logits/rejected": -1.9772014617919922,
      "logps/chosen": -242.79864501953125,
      "logps/rejected": -329.5115051269531,
      "loss": 0.0578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.13314151763916,
      "rewards/margins": 8.251870155334473,
      "rewards/rejected": -13.385011672973633,
      "step": 2069
    },
    {
      "epoch": 2.71,
      "learning_rate": 4.2040645794408445e-05,
      "logits/chosen": -2.1458396911621094,
      "logits/rejected": -2.4645326137542725,
      "logps/chosen": -261.87310791015625,
      "logps/rejected": -394.9166259765625,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.958534240722656,
      "rewards/margins": 8.967354774475098,
      "rewards/rejected": -13.92588996887207,
      "step": 2070
    },
    {
      "epoch": 2.71,
      "learning_rate": 4.2033022610530495e-05,
      "logits/chosen": -2.5229756832122803,
      "logits/rejected": -2.5427346229553223,
      "logps/chosen": -283.5986022949219,
      "logps/rejected": -337.8445129394531,
      "loss": 0.0473,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.727269172668457,
      "rewards/margins": 7.151669025421143,
      "rewards/rejected": -12.878936767578125,
      "step": 2071
    },
    {
      "epoch": 2.71,
      "learning_rate": 4.202539646966993e-05,
      "logits/chosen": -2.301621675491333,
      "logits/rejected": -2.393721103668213,
      "logps/chosen": -258.2679748535156,
      "logps/rejected": -371.85040283203125,
      "loss": 0.062,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.82385778427124,
      "rewards/margins": 8.38097095489502,
      "rewards/rejected": -13.204830169677734,
      "step": 2072
    },
    {
      "epoch": 2.71,
      "learning_rate": 4.201776737315069e-05,
      "logits/chosen": -1.9535908699035645,
      "logits/rejected": -1.902817726135254,
      "logps/chosen": -195.61341857910156,
      "logps/rejected": -286.3548889160156,
      "loss": 0.0976,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.179656505584717,
      "rewards/margins": 7.419997215270996,
      "rewards/rejected": -11.599653244018555,
      "step": 2073
    },
    {
      "epoch": 2.71,
      "learning_rate": 4.201013532229717e-05,
      "logits/chosen": -2.2434518337249756,
      "logits/rejected": -2.2938058376312256,
      "logps/chosen": -273.43939208984375,
      "logps/rejected": -355.3121643066406,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.7452874183654785,
      "rewards/margins": 8.803860664367676,
      "rewards/rejected": -13.549148559570312,
      "step": 2074
    },
    {
      "epoch": 2.72,
      "learning_rate": 4.200250031843436e-05,
      "logits/chosen": -2.147690773010254,
      "logits/rejected": -2.103121280670166,
      "logps/chosen": -252.52203369140625,
      "logps/rejected": -345.4867248535156,
      "loss": 0.0054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.055788516998291,
      "rewards/margins": 8.033418655395508,
      "rewards/rejected": -13.08920669555664,
      "step": 2075
    },
    {
      "epoch": 2.72,
      "learning_rate": 4.19948623628877e-05,
      "logits/chosen": -2.1535098552703857,
      "logits/rejected": -2.178994655609131,
      "logps/chosen": -201.6510009765625,
      "logps/rejected": -293.91864013671875,
      "loss": 0.0877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.277956962585449,
      "rewards/margins": 8.379180908203125,
      "rewards/rejected": -12.657137870788574,
      "step": 2076
    },
    {
      "epoch": 2.72,
      "learning_rate": 4.198722145698316e-05,
      "logits/chosen": -2.069924831390381,
      "logits/rejected": -2.08437180519104,
      "logps/chosen": -180.403564453125,
      "logps/rejected": -288.7211608886719,
      "loss": 0.0462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.470130920410156,
      "rewards/margins": 9.821898460388184,
      "rewards/rejected": -14.29202938079834,
      "step": 2077
    },
    {
      "epoch": 2.72,
      "learning_rate": 4.1979577602047225e-05,
      "logits/chosen": -1.9232547283172607,
      "logits/rejected": -1.9797210693359375,
      "logps/chosen": -221.45703125,
      "logps/rejected": -298.70947265625,
      "loss": 0.1112,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.912723064422607,
      "rewards/margins": 6.433436870574951,
      "rewards/rejected": -11.346160888671875,
      "step": 2078
    },
    {
      "epoch": 2.72,
      "learning_rate": 4.1971930799406886e-05,
      "logits/chosen": -1.878727674484253,
      "logits/rejected": -1.8984040021896362,
      "logps/chosen": -220.32061767578125,
      "logps/rejected": -301.4939880371094,
      "loss": 0.3477,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.683475494384766,
      "rewards/margins": 8.25886344909668,
      "rewards/rejected": -12.942338943481445,
      "step": 2079
    },
    {
      "epoch": 2.72,
      "learning_rate": 4.196428105038966e-05,
      "logits/chosen": -2.0797531604766846,
      "logits/rejected": -2.1228551864624023,
      "logps/chosen": -267.1173400878906,
      "logps/rejected": -339.40972900390625,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.964832782745361,
      "rewards/margins": 8.00648307800293,
      "rewards/rejected": -12.971315383911133,
      "step": 2080
    },
    {
      "epoch": 2.72,
      "learning_rate": 4.195662835632356e-05,
      "logits/chosen": -2.11684513092041,
      "logits/rejected": -2.075150728225708,
      "logps/chosen": -258.8525390625,
      "logps/rejected": -323.33953857421875,
      "loss": 0.1371,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.296041488647461,
      "rewards/margins": 6.962072849273682,
      "rewards/rejected": -12.2581148147583,
      "step": 2081
    },
    {
      "epoch": 2.72,
      "learning_rate": 4.194897271853712e-05,
      "logits/chosen": -2.0427944660186768,
      "logits/rejected": -2.0676662921905518,
      "logps/chosen": -270.5080871582031,
      "logps/rejected": -346.3038635253906,
      "loss": 0.0945,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.30461311340332,
      "rewards/margins": 7.249161720275879,
      "rewards/rejected": -12.553773880004883,
      "step": 2082
    },
    {
      "epoch": 2.73,
      "learning_rate": 4.194131413835937e-05,
      "logits/chosen": -2.061204433441162,
      "logits/rejected": -2.257317543029785,
      "logps/chosen": -209.44093322753906,
      "logps/rejected": -327.5050048828125,
      "loss": 0.0033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.970865249633789,
      "rewards/margins": 8.313035011291504,
      "rewards/rejected": -14.283899307250977,
      "step": 2083
    },
    {
      "epoch": 2.73,
      "learning_rate": 4.1933652617119884e-05,
      "logits/chosen": -2.4756479263305664,
      "logits/rejected": -2.410625457763672,
      "logps/chosen": -236.7713165283203,
      "logps/rejected": -275.470947265625,
      "loss": 0.1362,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.773379325866699,
      "rewards/margins": 5.622978210449219,
      "rewards/rejected": -10.396357536315918,
      "step": 2084
    },
    {
      "epoch": 2.73,
      "learning_rate": 4.19259881561487e-05,
      "logits/chosen": -2.2148780822753906,
      "logits/rejected": -2.1019504070281982,
      "logps/chosen": -210.04232788085938,
      "logps/rejected": -298.0214538574219,
      "loss": 0.0892,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.023033142089844,
      "rewards/margins": 8.117776870727539,
      "rewards/rejected": -13.140810012817383,
      "step": 2085
    },
    {
      "epoch": 2.73,
      "learning_rate": 4.1918320756776406e-05,
      "logits/chosen": -2.0880420207977295,
      "logits/rejected": -2.0963172912597656,
      "logps/chosen": -234.06309509277344,
      "logps/rejected": -323.31591796875,
      "loss": 0.0448,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.505844593048096,
      "rewards/margins": 8.520071029663086,
      "rewards/rejected": -13.025914192199707,
      "step": 2086
    },
    {
      "epoch": 2.73,
      "learning_rate": 4.1910650420334084e-05,
      "logits/chosen": -2.298971176147461,
      "logits/rejected": -2.3134219646453857,
      "logps/chosen": -253.9832763671875,
      "logps/rejected": -341.1661376953125,
      "loss": 0.0957,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.26953125,
      "rewards/margins": 7.074673652648926,
      "rewards/rejected": -13.344204902648926,
      "step": 2087
    },
    {
      "epoch": 2.73,
      "learning_rate": 4.190297714815332e-05,
      "logits/chosen": -2.1651523113250732,
      "logits/rejected": -2.3193411827087402,
      "logps/chosen": -243.05581665039062,
      "logps/rejected": -396.4129333496094,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.518342018127441,
      "rewards/margins": 10.284740447998047,
      "rewards/rejected": -15.803083419799805,
      "step": 2088
    },
    {
      "epoch": 2.73,
      "learning_rate": 4.189530094156622e-05,
      "logits/chosen": -2.185380220413208,
      "logits/rejected": -2.256591320037842,
      "logps/chosen": -275.3895568847656,
      "logps/rejected": -398.8544616699219,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.499395847320557,
      "rewards/margins": 9.482010841369629,
      "rewards/rejected": -14.981407165527344,
      "step": 2089
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.1887621801905396e-05,
      "logits/chosen": -2.2365474700927734,
      "logits/rejected": -2.258640766143799,
      "logps/chosen": -266.8921813964844,
      "logps/rejected": -392.3157653808594,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.301426887512207,
      "rewards/margins": 9.61983871459961,
      "rewards/rejected": -14.9212646484375,
      "step": 2090
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.187993973050397e-05,
      "logits/chosen": -2.1382174491882324,
      "logits/rejected": -2.1736865043640137,
      "logps/chosen": -223.3988037109375,
      "logps/rejected": -318.99798583984375,
      "loss": 0.0911,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.835805416107178,
      "rewards/margins": 8.29453182220459,
      "rewards/rejected": -14.130338668823242,
      "step": 2091
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.187225472869557e-05,
      "logits/chosen": -1.7872908115386963,
      "logits/rejected": -1.9052330255508423,
      "logps/chosen": -248.73348999023438,
      "logps/rejected": -378.89990234375,
      "loss": 0.0541,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.4027419090271,
      "rewards/margins": 8.380805969238281,
      "rewards/rejected": -13.783547401428223,
      "step": 2092
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.186456679781434e-05,
      "logits/chosen": -2.0523717403411865,
      "logits/rejected": -2.1558446884155273,
      "logps/chosen": -246.02601623535156,
      "logps/rejected": -373.6942138671875,
      "loss": 0.0447,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.165873050689697,
      "rewards/margins": 8.869769096374512,
      "rewards/rejected": -14.035642623901367,
      "step": 2093
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.1856875939194917e-05,
      "logits/chosen": -2.036025047302246,
      "logits/rejected": -2.0010640621185303,
      "logps/chosen": -219.66464233398438,
      "logps/rejected": -335.1212463378906,
      "loss": 0.0489,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.9590160846710205,
      "rewards/margins": 8.931244850158691,
      "rewards/rejected": -12.890260696411133,
      "step": 2094
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.1849182154172475e-05,
      "logits/chosen": -2.077622890472412,
      "logits/rejected": -2.213207483291626,
      "logps/chosen": -234.46327209472656,
      "logps/rejected": -375.966552734375,
      "loss": 0.0858,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.090786457061768,
      "rewards/margins": 9.148885726928711,
      "rewards/rejected": -15.239669799804688,
      "step": 2095
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.184148544408265e-05,
      "logits/chosen": -1.9233466386795044,
      "logits/rejected": -1.9716625213623047,
      "logps/chosen": -260.3994445800781,
      "logps/rejected": -336.7559509277344,
      "loss": 0.0941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.399205684661865,
      "rewards/margins": 6.785099029541016,
      "rewards/rejected": -12.184305191040039,
      "step": 2096
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.183378581026164e-05,
      "logits/chosen": -1.9934498071670532,
      "logits/rejected": -2.0427863597869873,
      "logps/chosen": -233.28619384765625,
      "logps/rejected": -308.950439453125,
      "loss": 0.0463,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.482067108154297,
      "rewards/margins": 7.9834465980529785,
      "rewards/rejected": -13.4655122756958,
      "step": 2097
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.182608325404611e-05,
      "logits/chosen": -2.2864651679992676,
      "logits/rejected": -2.3209176063537598,
      "logps/chosen": -228.93246459960938,
      "logps/rejected": -322.6388244628906,
      "loss": 0.0869,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.583909034729004,
      "rewards/margins": 9.239480972290039,
      "rewards/rejected": -12.823390007019043,
      "step": 2098
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.1818377776773255e-05,
      "logits/chosen": -2.0176432132720947,
      "logits/rejected": -2.086442232131958,
      "logps/chosen": -198.83099365234375,
      "logps/rejected": -302.553466796875,
      "loss": 0.0441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.565010070800781,
      "rewards/margins": 8.124502182006836,
      "rewards/rejected": -13.689510345458984,
      "step": 2099
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.181066937978075e-05,
      "logits/chosen": -2.3962371349334717,
      "logits/rejected": -2.3214006423950195,
      "logps/chosen": -244.8721923828125,
      "logps/rejected": -286.8533935546875,
      "loss": 0.2173,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.8245086669921875,
      "rewards/margins": 6.910382270812988,
      "rewards/rejected": -10.734890937805176,
      "step": 2100
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.180295806440682e-05,
      "logits/chosen": -1.9995003938674927,
      "logits/rejected": -2.089867115020752,
      "logps/chosen": -198.92347717285156,
      "logps/rejected": -302.6363830566406,
      "loss": 0.0461,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.106322765350342,
      "rewards/margins": 7.756969928741455,
      "rewards/rejected": -12.863292694091797,
      "step": 2101
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.179524383199016e-05,
      "logits/chosen": -1.8776270151138306,
      "logits/rejected": -2.059648036956787,
      "logps/chosen": -245.83056640625,
      "logps/rejected": -301.4813537597656,
      "loss": 0.0485,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.165894031524658,
      "rewards/margins": 7.827360153198242,
      "rewards/rejected": -12.993253707885742,
      "step": 2102
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.178752668386997e-05,
      "logits/chosen": -2.220716953277588,
      "logits/rejected": -2.236607313156128,
      "logps/chosen": -242.704833984375,
      "logps/rejected": -326.8563232421875,
      "loss": 0.0522,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.967065811157227,
      "rewards/margins": 7.383824348449707,
      "rewards/rejected": -13.350890159606934,
      "step": 2103
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.1779806621386e-05,
      "logits/chosen": -1.9287221431732178,
      "logits/rejected": -1.9850231409072876,
      "logps/chosen": -232.6581268310547,
      "logps/rejected": -301.498046875,
      "loss": 0.0957,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.314054489135742,
      "rewards/margins": 6.8057403564453125,
      "rewards/rejected": -12.119794845581055,
      "step": 2104
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.1772083645878444e-05,
      "logits/chosen": -2.194317579269409,
      "logits/rejected": -2.2162156105041504,
      "logps/chosen": -189.45413208007812,
      "logps/rejected": -277.4154052734375,
      "loss": 0.0177,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.497744560241699,
      "rewards/margins": 7.991286277770996,
      "rewards/rejected": -12.489030838012695,
      "step": 2105
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.176435775868805e-05,
      "logits/chosen": -2.1279244422912598,
      "logits/rejected": -2.165595769882202,
      "logps/chosen": -273.2611999511719,
      "logps/rejected": -371.8103332519531,
      "loss": 0.1316,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.4172234535217285,
      "rewards/margins": 8.07325553894043,
      "rewards/rejected": -12.490479469299316,
      "step": 2106
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.1756628961156064e-05,
      "logits/chosen": -1.984653115272522,
      "logits/rejected": -2.1006929874420166,
      "logps/chosen": -293.9515686035156,
      "logps/rejected": -393.9189453125,
      "loss": 0.0439,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.42185115814209,
      "rewards/margins": 8.856366157531738,
      "rewards/rejected": -15.278217315673828,
      "step": 2107
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.174889725462421e-05,
      "logits/chosen": -2.2485594749450684,
      "logits/rejected": -2.168731451034546,
      "logps/chosen": -238.37648010253906,
      "logps/rejected": -324.60223388671875,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.376681804656982,
      "rewards/margins": 9.401522636413574,
      "rewards/rejected": -14.778203964233398,
      "step": 2108
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.1741162640434736e-05,
      "logits/chosen": -1.9732999801635742,
      "logits/rejected": -2.150045871734619,
      "logps/chosen": -243.09902954101562,
      "logps/rejected": -379.98779296875,
      "loss": 0.0034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.749682903289795,
      "rewards/margins": 9.43840217590332,
      "rewards/rejected": -15.188084602355957,
      "step": 2109
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.17334251199304e-05,
      "logits/chosen": -2.0670886039733887,
      "logits/rejected": -1.9970879554748535,
      "logps/chosen": -212.93011474609375,
      "logps/rejected": -290.45220947265625,
      "loss": 0.0594,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.849340438842773,
      "rewards/margins": 8.22266960144043,
      "rewards/rejected": -13.072010040283203,
      "step": 2110
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.172568469445446e-05,
      "logits/chosen": -2.146941661834717,
      "logits/rejected": -2.128082036972046,
      "logps/chosen": -288.23114013671875,
      "logps/rejected": -340.2186584472656,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.491695404052734,
      "rewards/margins": 8.292865753173828,
      "rewards/rejected": -14.784561157226562,
      "step": 2111
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.171794136535067e-05,
      "logits/chosen": -2.09254789352417,
      "logits/rejected": -2.106165885925293,
      "logps/chosen": -246.88629150390625,
      "logps/rejected": -331.7405700683594,
      "loss": 0.049,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.748941898345947,
      "rewards/margins": 8.975343704223633,
      "rewards/rejected": -14.724285125732422,
      "step": 2112
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.17101951339633e-05,
      "logits/chosen": -1.9903786182403564,
      "logits/rejected": -1.9985647201538086,
      "logps/chosen": -216.80055236816406,
      "logps/rejected": -314.84552001953125,
      "loss": 0.0941,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.6483473777771,
      "rewards/margins": 7.770323276519775,
      "rewards/rejected": -12.418669700622559,
      "step": 2113
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.170244600163712e-05,
      "logits/chosen": -2.100982904434204,
      "logits/rejected": -2.082028865814209,
      "logps/chosen": -215.8398895263672,
      "logps/rejected": -252.7769317626953,
      "loss": 0.0464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.179358959197998,
      "rewards/margins": 7.598566055297852,
      "rewards/rejected": -11.777924537658691,
      "step": 2114
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.169469396971739e-05,
      "logits/chosen": -2.2202951908111572,
      "logits/rejected": -2.112492084503174,
      "logps/chosen": -271.6481018066406,
      "logps/rejected": -403.6411437988281,
      "loss": 0.0063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.043911933898926,
      "rewards/margins": 8.47602367401123,
      "rewards/rejected": -14.519936561584473,
      "step": 2115
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.16869390395499e-05,
      "logits/chosen": -2.1215877532958984,
      "logits/rejected": -2.1512339115142822,
      "logps/chosen": -226.36605834960938,
      "logps/rejected": -314.9847412109375,
      "loss": 0.0776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.515926837921143,
      "rewards/margins": 7.796207904815674,
      "rewards/rejected": -13.3121337890625,
      "step": 2116
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.167918121248092e-05,
      "logits/chosen": -2.2617945671081543,
      "logits/rejected": -2.2887606620788574,
      "logps/chosen": -289.6715087890625,
      "logps/rejected": -347.873291015625,
      "loss": 0.0192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.545827865600586,
      "rewards/margins": 9.474298477172852,
      "rewards/rejected": -16.020126342773438,
      "step": 2117
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.1671420489857236e-05,
      "logits/chosen": -2.0493292808532715,
      "logits/rejected": -2.0911078453063965,
      "logps/chosen": -249.25364685058594,
      "logps/rejected": -380.7524719238281,
      "loss": 0.0518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.965695858001709,
      "rewards/margins": 8.083683013916016,
      "rewards/rejected": -15.049379348754883,
      "step": 2118
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.166365687302614e-05,
      "logits/chosen": -1.9570484161376953,
      "logits/rejected": -1.8582700490951538,
      "logps/chosen": -211.3433074951172,
      "logps/rejected": -304.293701171875,
      "loss": 0.0492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.687077045440674,
      "rewards/margins": 8.289334297180176,
      "rewards/rejected": -13.976410865783691,
      "step": 2119
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.1655890363335394e-05,
      "logits/chosen": -2.077530860900879,
      "logits/rejected": -2.1311354637145996,
      "logps/chosen": -252.41888427734375,
      "logps/rejected": -354.2386474609375,
      "loss": 0.0439,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.369992256164551,
      "rewards/margins": 10.087926864624023,
      "rewards/rejected": -16.457918167114258,
      "step": 2120
    },
    {
      "epoch": 2.78,
      "learning_rate": 4.164812096213331e-05,
      "logits/chosen": -1.9602264165878296,
      "logits/rejected": -2.0098724365234375,
      "logps/chosen": -237.26214599609375,
      "logps/rejected": -300.9123840332031,
      "loss": 0.0886,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.2542524337768555,
      "rewards/margins": 6.468628406524658,
      "rewards/rejected": -12.722880363464355,
      "step": 2121
    },
    {
      "epoch": 2.78,
      "learning_rate": 4.164034867076866e-05,
      "logits/chosen": -2.0425612926483154,
      "logits/rejected": -2.056826114654541,
      "logps/chosen": -228.2157745361328,
      "logps/rejected": -342.6788024902344,
      "loss": 0.0438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.24842643737793,
      "rewards/margins": 10.305375099182129,
      "rewards/rejected": -17.553800582885742,
      "step": 2122
    },
    {
      "epoch": 2.78,
      "learning_rate": 4.1632573490590754e-05,
      "logits/chosen": -1.9484583139419556,
      "logits/rejected": -1.9971625804901123,
      "logps/chosen": -205.79505920410156,
      "logps/rejected": -284.4195556640625,
      "loss": 0.1798,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.692427158355713,
      "rewards/margins": 8.066761016845703,
      "rewards/rejected": -13.759187698364258,
      "step": 2123
    },
    {
      "epoch": 2.78,
      "learning_rate": 4.162479542294938e-05,
      "logits/chosen": -2.029684543609619,
      "logits/rejected": -2.0470805168151855,
      "logps/chosen": -287.4469909667969,
      "logps/rejected": -360.28680419921875,
      "loss": 0.0261,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -8.992703437805176,
      "rewards/margins": 8.311714172363281,
      "rewards/rejected": -17.30441665649414,
      "step": 2124
    },
    {
      "epoch": 2.78,
      "learning_rate": 4.1617014469194836e-05,
      "logits/chosen": -1.9437496662139893,
      "logits/rejected": -1.940740942955017,
      "logps/chosen": -280.3132019042969,
      "logps/rejected": -343.2749328613281,
      "loss": 0.2108,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.682542324066162,
      "rewards/margins": 8.812956809997559,
      "rewards/rejected": -16.495498657226562,
      "step": 2125
    },
    {
      "epoch": 2.78,
      "learning_rate": 4.160923063067791e-05,
      "logits/chosen": -1.6997432708740234,
      "logits/rejected": -1.8715956211090088,
      "logps/chosen": -235.1721649169922,
      "logps/rejected": -387.16510009765625,
      "loss": 0.1111,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.875338554382324,
      "rewards/margins": 9.192512512207031,
      "rewards/rejected": -16.067852020263672,
      "step": 2126
    },
    {
      "epoch": 2.78,
      "learning_rate": 4.1601443908749904e-05,
      "logits/chosen": -1.8663256168365479,
      "logits/rejected": -1.9868435859680176,
      "logps/chosen": -274.02374267578125,
      "logps/rejected": -363.5721435546875,
      "loss": 0.0043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.386130332946777,
      "rewards/margins": 8.717323303222656,
      "rewards/rejected": -15.103453636169434,
      "step": 2127
    },
    {
      "epoch": 2.78,
      "learning_rate": 4.159365430476261e-05,
      "logits/chosen": -1.8189918994903564,
      "logits/rejected": -1.8256369829177856,
      "logps/chosen": -236.00656127929688,
      "logps/rejected": -336.5587158203125,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.672895431518555,
      "rewards/margins": 9.486919403076172,
      "rewards/rejected": -15.15981388092041,
      "step": 2128
    },
    {
      "epoch": 2.79,
      "learning_rate": 4.158586182006834e-05,
      "logits/chosen": -1.9643597602844238,
      "logits/rejected": -1.9393858909606934,
      "logps/chosen": -276.75238037109375,
      "logps/rejected": -361.45660400390625,
      "loss": 0.0755,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.428586959838867,
      "rewards/margins": 9.840063095092773,
      "rewards/rejected": -16.26865005493164,
      "step": 2129
    },
    {
      "epoch": 2.79,
      "learning_rate": 4.157806645601988e-05,
      "logits/chosen": -2.0720882415771484,
      "logits/rejected": -2.129720449447632,
      "logps/chosen": -305.78948974609375,
      "logps/rejected": -402.7105712890625,
      "loss": 0.1377,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.76812744140625,
      "rewards/margins": 6.746726989746094,
      "rewards/rejected": -14.514853477478027,
      "step": 2130
    },
    {
      "epoch": 2.79,
      "learning_rate": 4.1570268213970534e-05,
      "logits/chosen": -1.9224815368652344,
      "logits/rejected": -1.9703435897827148,
      "logps/chosen": -272.35504150390625,
      "logps/rejected": -383.93963623046875,
      "loss": 0.0487,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.817448616027832,
      "rewards/margins": 9.154064178466797,
      "rewards/rejected": -15.971511840820312,
      "step": 2131
    },
    {
      "epoch": 2.79,
      "learning_rate": 4.15624670952741e-05,
      "logits/chosen": -2.1934380531311035,
      "logits/rejected": -2.250469923019409,
      "logps/chosen": -285.520263671875,
      "logps/rejected": -397.2008361816406,
      "loss": 0.1347,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -7.1292219161987305,
      "rewards/margins": 8.519986152648926,
      "rewards/rejected": -15.649208068847656,
      "step": 2132
    },
    {
      "epoch": 2.79,
      "learning_rate": 4.155466310128487e-05,
      "logits/chosen": -2.1712989807128906,
      "logits/rejected": -2.1715948581695557,
      "logps/chosen": -237.79794311523438,
      "logps/rejected": -321.9048767089844,
      "loss": 0.0453,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.765093803405762,
      "rewards/margins": 8.670004844665527,
      "rewards/rejected": -14.435096740722656,
      "step": 2133
    },
    {
      "epoch": 2.79,
      "learning_rate": 4.1546856233357645e-05,
      "logits/chosen": -2.097083806991577,
      "logits/rejected": -2.052821397781372,
      "logps/chosen": -240.02984619140625,
      "logps/rejected": -311.79925537109375,
      "loss": 0.0614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.495797634124756,
      "rewards/margins": 7.645787239074707,
      "rewards/rejected": -14.141584396362305,
      "step": 2134
    },
    {
      "epoch": 2.79,
      "learning_rate": 4.153904649284771e-05,
      "logits/chosen": -2.127629518508911,
      "logits/rejected": -2.191781520843506,
      "logps/chosen": -333.68499755859375,
      "logps/rejected": -373.86328125,
      "loss": 0.048,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.099422931671143,
      "rewards/margins": 8.105523109436035,
      "rewards/rejected": -15.204946517944336,
      "step": 2135
    },
    {
      "epoch": 2.8,
      "learning_rate": 4.1531233881110875e-05,
      "logits/chosen": -1.9232991933822632,
      "logits/rejected": -2.016394853591919,
      "logps/chosen": -231.78211975097656,
      "logps/rejected": -310.239501953125,
      "loss": 0.0507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.600955963134766,
      "rewards/margins": 8.300896644592285,
      "rewards/rejected": -13.901851654052734,
      "step": 2136
    },
    {
      "epoch": 2.8,
      "learning_rate": 4.152341839950342e-05,
      "logits/chosen": -1.766890048980713,
      "logits/rejected": -1.7369945049285889,
      "logps/chosen": -266.9042053222656,
      "logps/rejected": -396.30291748046875,
      "loss": 0.2204,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -7.4335408210754395,
      "rewards/margins": 9.401388168334961,
      "rewards/rejected": -16.834928512573242,
      "step": 2137
    },
    {
      "epoch": 2.8,
      "learning_rate": 4.151560004938214e-05,
      "logits/chosen": -1.7209428548812866,
      "logits/rejected": -1.8653645515441895,
      "logps/chosen": -246.06756591796875,
      "logps/rejected": -348.609130859375,
      "loss": 0.133,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.485172271728516,
      "rewards/margins": 8.861818313598633,
      "rewards/rejected": -14.346990585327148,
      "step": 2138
    },
    {
      "epoch": 2.8,
      "learning_rate": 4.1507778832104326e-05,
      "logits/chosen": -2.0044240951538086,
      "logits/rejected": -2.0473899841308594,
      "logps/chosen": -289.8185729980469,
      "logps/rejected": -395.8297424316406,
      "loss": 0.0457,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.868648052215576,
      "rewards/margins": 9.16476058959961,
      "rewards/rejected": -16.033409118652344,
      "step": 2139
    },
    {
      "epoch": 2.8,
      "learning_rate": 4.149995474902776e-05,
      "logits/chosen": -2.0526278018951416,
      "logits/rejected": -2.1411993503570557,
      "logps/chosen": -212.82223510742188,
      "logps/rejected": -308.5693359375,
      "loss": 0.0115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.235255718231201,
      "rewards/margins": 8.20628547668457,
      "rewards/rejected": -13.441540718078613,
      "step": 2140
    },
    {
      "epoch": 2.8,
      "learning_rate": 4.149212780151073e-05,
      "logits/chosen": -2.014158248901367,
      "logits/rejected": -2.1073756217956543,
      "logps/chosen": -211.17178344726562,
      "logps/rejected": -347.7945251464844,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.059771537780762,
      "rewards/margins": 10.15962028503418,
      "rewards/rejected": -16.219392776489258,
      "step": 2141
    },
    {
      "epoch": 2.8,
      "learning_rate": 4.1484297990912e-05,
      "logits/chosen": -2.045736312866211,
      "logits/rejected": -2.090200424194336,
      "logps/chosen": -269.7496337890625,
      "logps/rejected": -362.4156799316406,
      "loss": 0.0069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.597815990447998,
      "rewards/margins": 7.994004726409912,
      "rewards/rejected": -15.591819763183594,
      "step": 2142
    },
    {
      "epoch": 2.8,
      "learning_rate": 4.1476465318590876e-05,
      "logits/chosen": -2.0298562049865723,
      "logits/rejected": -2.122493267059326,
      "logps/chosen": -218.05921936035156,
      "logps/rejected": -343.3038635253906,
      "loss": 0.0487,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.693921089172363,
      "rewards/margins": 8.963953018188477,
      "rewards/rejected": -14.657873153686523,
      "step": 2143
    },
    {
      "epoch": 2.81,
      "learning_rate": 4.146862978590712e-05,
      "logits/chosen": -1.8342090845108032,
      "logits/rejected": -1.959254503250122,
      "logps/chosen": -264.7921142578125,
      "logps/rejected": -354.5204772949219,
      "loss": 0.0889,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.0057196617126465,
      "rewards/margins": 8.22575855255127,
      "rewards/rejected": -14.231477737426758,
      "step": 2144
    },
    {
      "epoch": 2.81,
      "learning_rate": 4.1460791394221e-05,
      "logits/chosen": -1.7905089855194092,
      "logits/rejected": -1.8561276197433472,
      "logps/chosen": -291.75823974609375,
      "logps/rejected": -356.01495361328125,
      "loss": 0.2051,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -5.946643352508545,
      "rewards/margins": 6.129225730895996,
      "rewards/rejected": -12.0758695602417,
      "step": 2145
    },
    {
      "epoch": 2.81,
      "learning_rate": 4.1452950144893274e-05,
      "logits/chosen": -2.0241174697875977,
      "logits/rejected": -2.05717396736145,
      "logps/chosen": -253.92105102539062,
      "logps/rejected": -348.9212341308594,
      "loss": 0.0081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.968001365661621,
      "rewards/margins": 9.411520957946777,
      "rewards/rejected": -16.3795223236084,
      "step": 2146
    },
    {
      "epoch": 2.81,
      "learning_rate": 4.144510603928523e-05,
      "logits/chosen": -1.9531909227371216,
      "logits/rejected": -2.044700860977173,
      "logps/chosen": -290.6370544433594,
      "logps/rejected": -353.8155517578125,
      "loss": 0.0503,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.445758819580078,
      "rewards/margins": 7.122509479522705,
      "rewards/rejected": -14.568268775939941,
      "step": 2147
    },
    {
      "epoch": 2.81,
      "learning_rate": 4.143725907875862e-05,
      "logits/chosen": -1.7878367900848389,
      "logits/rejected": -1.8614583015441895,
      "logps/chosen": -290.497314453125,
      "logps/rejected": -393.19500732421875,
      "loss": 0.0872,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.828826904296875,
      "rewards/margins": 8.184590339660645,
      "rewards/rejected": -14.013418197631836,
      "step": 2148
    },
    {
      "epoch": 2.81,
      "learning_rate": 4.142940926467568e-05,
      "logits/chosen": -1.9906575679779053,
      "logits/rejected": -1.9775463342666626,
      "logps/chosen": -212.44091796875,
      "logps/rejected": -325.2464294433594,
      "loss": 0.0876,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.200534820556641,
      "rewards/margins": 7.944638252258301,
      "rewards/rejected": -14.145173072814941,
      "step": 2149
    },
    {
      "epoch": 2.81,
      "learning_rate": 4.142155659839919e-05,
      "logits/chosen": -2.054295063018799,
      "logits/rejected": -2.074632406234741,
      "logps/chosen": -193.02542114257812,
      "logps/rejected": -294.2581787109375,
      "loss": 0.0285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.436005115509033,
      "rewards/margins": 7.639278411865234,
      "rewards/rejected": -13.07528305053711,
      "step": 2150
    },
    {
      "epoch": 2.81,
      "learning_rate": 4.141370108129238e-05,
      "logits/chosen": -2.0766634941101074,
      "logits/rejected": -2.038219451904297,
      "logps/chosen": -236.00892639160156,
      "logps/rejected": -347.83319091796875,
      "loss": 0.087,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.99525260925293,
      "rewards/margins": 8.909188270568848,
      "rewards/rejected": -13.904440879821777,
      "step": 2151
    },
    {
      "epoch": 2.82,
      "learning_rate": 4.1405842714718985e-05,
      "logits/chosen": -2.1349785327911377,
      "logits/rejected": -2.10587739944458,
      "logps/chosen": -295.6947326660156,
      "logps/rejected": -348.4321594238281,
      "loss": 0.0535,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.5709710121154785,
      "rewards/margins": 6.291179656982422,
      "rewards/rejected": -11.862152099609375,
      "step": 2152
    },
    {
      "epoch": 2.82,
      "learning_rate": 4.139798150004325e-05,
      "logits/chosen": -1.93121337890625,
      "logits/rejected": -2.0886569023132324,
      "logps/chosen": -217.17417907714844,
      "logps/rejected": -329.839111328125,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.0091552734375,
      "rewards/margins": 9.298357963562012,
      "rewards/rejected": -15.307512283325195,
      "step": 2153
    },
    {
      "epoch": 2.82,
      "learning_rate": 4.139011743862991e-05,
      "logits/chosen": -2.001394510269165,
      "logits/rejected": -2.1222944259643555,
      "logps/chosen": -207.75503540039062,
      "logps/rejected": -310.826171875,
      "loss": 0.1308,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.948476314544678,
      "rewards/margins": 8.621466636657715,
      "rewards/rejected": -13.56994342803955,
      "step": 2154
    },
    {
      "epoch": 2.82,
      "learning_rate": 4.138225053184417e-05,
      "logits/chosen": -2.2703278064727783,
      "logits/rejected": -2.2544658184051514,
      "logps/chosen": -218.79788208007812,
      "logps/rejected": -288.935791015625,
      "loss": 0.0203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.186158180236816,
      "rewards/margins": 6.715668678283691,
      "rewards/rejected": -11.901827812194824,
      "step": 2155
    },
    {
      "epoch": 2.82,
      "learning_rate": 4.137438078105176e-05,
      "logits/chosen": -2.1223785877227783,
      "logits/rejected": -2.1393320560455322,
      "logps/chosen": -240.15248107910156,
      "logps/rejected": -336.35992431640625,
      "loss": 0.0464,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.761418342590332,
      "rewards/margins": 8.959957122802734,
      "rewards/rejected": -13.72137451171875,
      "step": 2156
    },
    {
      "epoch": 2.82,
      "learning_rate": 4.1366508187618895e-05,
      "logits/chosen": -1.9917843341827393,
      "logits/rejected": -1.9294066429138184,
      "logps/chosen": -202.04116821289062,
      "logps/rejected": -256.02581787109375,
      "loss": 0.0574,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.986374855041504,
      "rewards/margins": 6.414172172546387,
      "rewards/rejected": -11.40054702758789,
      "step": 2157
    },
    {
      "epoch": 2.82,
      "learning_rate": 4.135863275291229e-05,
      "logits/chosen": -1.9927433729171753,
      "logits/rejected": -1.9758617877960205,
      "logps/chosen": -264.622314453125,
      "logps/rejected": -306.6430358886719,
      "loss": 0.0157,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.7758002281188965,
      "rewards/margins": 6.953046798706055,
      "rewards/rejected": -12.72884750366211,
      "step": 2158
    },
    {
      "epoch": 2.83,
      "learning_rate": 4.1350754478299114e-05,
      "logits/chosen": -2.1066510677337646,
      "logits/rejected": -2.15097713470459,
      "logps/chosen": -297.3056335449219,
      "logps/rejected": -377.4056396484375,
      "loss": 0.0444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.846955299377441,
      "rewards/margins": 8.380105018615723,
      "rewards/rejected": -13.227059364318848,
      "step": 2159
    },
    {
      "epoch": 2.83,
      "learning_rate": 4.1342873365147074e-05,
      "logits/chosen": -1.971630573272705,
      "logits/rejected": -1.931064486503601,
      "logps/chosen": -307.5804748535156,
      "logps/rejected": -372.26531982421875,
      "loss": 0.0626,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.562823295593262,
      "rewards/margins": 8.449759483337402,
      "rewards/rejected": -14.01258373260498,
      "step": 2160
    },
    {
      "epoch": 2.83,
      "learning_rate": 4.1334989414824364e-05,
      "logits/chosen": -1.9147617816925049,
      "logits/rejected": -1.9771555662155151,
      "logps/chosen": -235.437744140625,
      "logps/rejected": -322.82086181640625,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.258989334106445,
      "rewards/margins": 8.508306503295898,
      "rewards/rejected": -12.76729679107666,
      "step": 2161
    },
    {
      "epoch": 2.83,
      "learning_rate": 4.1327102628699646e-05,
      "logits/chosen": -1.9146307706832886,
      "logits/rejected": -1.8994238376617432,
      "logps/chosen": -223.66262817382812,
      "logps/rejected": -278.8446350097656,
      "loss": 0.087,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.353767156600952,
      "rewards/margins": 8.824553489685059,
      "rewards/rejected": -12.178321838378906,
      "step": 2162
    },
    {
      "epoch": 2.83,
      "learning_rate": 4.1319213008142106e-05,
      "logits/chosen": -1.9332091808319092,
      "logits/rejected": -2.0296545028686523,
      "logps/chosen": -244.3536376953125,
      "logps/rejected": -288.45928955078125,
      "loss": 0.047,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.429356575012207,
      "rewards/margins": 7.586400985717773,
      "rewards/rejected": -11.015758514404297,
      "step": 2163
    },
    {
      "epoch": 2.83,
      "learning_rate": 4.131132055452138e-05,
      "logits/chosen": -2.0869193077087402,
      "logits/rejected": -2.067127227783203,
      "logps/chosen": -226.43116760253906,
      "logps/rejected": -282.193359375,
      "loss": 0.0718,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.105947971343994,
      "rewards/margins": 7.346132278442383,
      "rewards/rejected": -11.452079772949219,
      "step": 2164
    },
    {
      "epoch": 2.83,
      "learning_rate": 4.130342526920765e-05,
      "logits/chosen": -2.236833333969116,
      "logits/rejected": -2.1940701007843018,
      "logps/chosen": -336.6755676269531,
      "logps/rejected": -400.3515319824219,
      "loss": 0.0466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.350348472595215,
      "rewards/margins": 8.958721160888672,
      "rewards/rejected": -13.309070587158203,
      "step": 2165
    },
    {
      "epoch": 2.83,
      "learning_rate": 4.129552715357153e-05,
      "logits/chosen": -1.699863314628601,
      "logits/rejected": -1.746483325958252,
      "logps/chosen": -229.27069091796875,
      "logps/rejected": -304.8611755371094,
      "loss": 0.1094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.467954635620117,
      "rewards/margins": 7.739739418029785,
      "rewards/rejected": -11.207694053649902,
      "step": 2166
    },
    {
      "epoch": 2.84,
      "learning_rate": 4.128762620898419e-05,
      "logits/chosen": -2.0914857387542725,
      "logits/rejected": -2.136545181274414,
      "logps/chosen": -275.3821716308594,
      "logps/rejected": -331.4245910644531,
      "loss": 0.0057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.5834550857543945,
      "rewards/margins": 6.8916544914245605,
      "rewards/rejected": -11.47511100769043,
      "step": 2167
    },
    {
      "epoch": 2.84,
      "learning_rate": 4.1279722436817226e-05,
      "logits/chosen": -1.9021196365356445,
      "logits/rejected": -2.0389928817749023,
      "logps/chosen": -189.16964721679688,
      "logps/rejected": -258.6311340332031,
      "loss": 0.0919,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.3057665824890137,
      "rewards/margins": 7.311253547668457,
      "rewards/rejected": -10.617019653320312,
      "step": 2168
    },
    {
      "epoch": 2.84,
      "learning_rate": 4.1271815838442774e-05,
      "logits/chosen": -1.6419003009796143,
      "logits/rejected": -1.6636180877685547,
      "logps/chosen": -275.1500244140625,
      "logps/rejected": -343.6715087890625,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.7763137817382812,
      "rewards/margins": 9.571589469909668,
      "rewards/rejected": -13.347904205322266,
      "step": 2169
    },
    {
      "epoch": 2.84,
      "learning_rate": 4.126390641523344e-05,
      "logits/chosen": -2.1074090003967285,
      "logits/rejected": -2.1533889770507812,
      "logps/chosen": -220.68756103515625,
      "logps/rejected": -292.22259521484375,
      "loss": 0.0455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.995213031768799,
      "rewards/margins": 7.116742134094238,
      "rewards/rejected": -11.111954689025879,
      "step": 2170
    },
    {
      "epoch": 2.84,
      "learning_rate": 4.125599416856232e-05,
      "logits/chosen": -1.9926451444625854,
      "logits/rejected": -1.931626319885254,
      "logps/chosen": -232.9684295654297,
      "logps/rejected": -290.0226745605469,
      "loss": 0.0133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.7480080127716064,
      "rewards/margins": 7.809638500213623,
      "rewards/rejected": -11.557646751403809,
      "step": 2171
    },
    {
      "epoch": 2.84,
      "learning_rate": 4.1248079099803004e-05,
      "logits/chosen": -2.080148458480835,
      "logits/rejected": -2.0510504245758057,
      "logps/chosen": -271.93731689453125,
      "logps/rejected": -353.4999694824219,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.3109636306762695,
      "rewards/margins": 10.576560974121094,
      "rewards/rejected": -14.88752269744873,
      "step": 2172
    },
    {
      "epoch": 2.84,
      "learning_rate": 4.124016121032957e-05,
      "logits/chosen": -2.082777500152588,
      "logits/rejected": -2.1824376583099365,
      "logps/chosen": -190.74664306640625,
      "logps/rejected": -261.811767578125,
      "loss": 0.1759,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.506443500518799,
      "rewards/margins": 6.029775619506836,
      "rewards/rejected": -9.536218643188477,
      "step": 2173
    },
    {
      "epoch": 2.85,
      "learning_rate": 4.12322405015166e-05,
      "logits/chosen": -2.0977468490600586,
      "logits/rejected": -1.9874368906021118,
      "logps/chosen": -198.05026245117188,
      "logps/rejected": -290.55340576171875,
      "loss": 0.1391,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.6181581020355225,
      "rewards/margins": 6.918950080871582,
      "rewards/rejected": -9.537108421325684,
      "step": 2174
    },
    {
      "epoch": 2.85,
      "learning_rate": 4.122431697473912e-05,
      "logits/chosen": -1.9559710025787354,
      "logits/rejected": -2.025653839111328,
      "logps/chosen": -288.446533203125,
      "logps/rejected": -345.2745361328125,
      "loss": 0.0913,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.877231121063232,
      "rewards/margins": 6.645547389984131,
      "rewards/rejected": -11.522778511047363,
      "step": 2175
    },
    {
      "epoch": 2.85,
      "learning_rate": 4.12163906313727e-05,
      "logits/chosen": -2.0126967430114746,
      "logits/rejected": -2.1336889266967773,
      "logps/chosen": -253.75491333007812,
      "logps/rejected": -331.97210693359375,
      "loss": 0.0436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.9921789169311523,
      "rewards/margins": 9.469217300415039,
      "rewards/rejected": -13.461396217346191,
      "step": 2176
    },
    {
      "epoch": 2.85,
      "learning_rate": 4.120846147279339e-05,
      "logits/chosen": -2.1571877002716064,
      "logits/rejected": -2.1231818199157715,
      "logps/chosen": -262.9522705078125,
      "logps/rejected": -369.7544860839844,
      "loss": 0.0023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.100529670715332,
      "rewards/margins": 8.334796905517578,
      "rewards/rejected": -13.435324668884277,
      "step": 2177
    },
    {
      "epoch": 2.85,
      "learning_rate": 4.120052950037769e-05,
      "logits/chosen": -1.8106213808059692,
      "logits/rejected": -2.013967514038086,
      "logps/chosen": -216.9224090576172,
      "logps/rejected": -329.19482421875,
      "loss": 0.0072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.862124443054199,
      "rewards/margins": 9.059403419494629,
      "rewards/rejected": -13.921527862548828,
      "step": 2178
    },
    {
      "epoch": 2.85,
      "learning_rate": 4.119259471550262e-05,
      "logits/chosen": -1.8358765840530396,
      "logits/rejected": -1.9351555109024048,
      "logps/chosen": -215.11273193359375,
      "logps/rejected": -340.1754150390625,
      "loss": 0.0455,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.384608268737793,
      "rewards/margins": 8.696837425231934,
      "rewards/rejected": -13.081446647644043,
      "step": 2179
    },
    {
      "epoch": 2.85,
      "learning_rate": 4.118465711954569e-05,
      "logits/chosen": -2.379478931427002,
      "logits/rejected": -2.270138740539551,
      "logps/chosen": -311.94403076171875,
      "logps/rejected": -377.7488708496094,
      "loss": 0.049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.311163902282715,
      "rewards/margins": 7.913090229034424,
      "rewards/rejected": -12.224254608154297,
      "step": 2180
    },
    {
      "epoch": 2.85,
      "learning_rate": 4.117671671388489e-05,
      "logits/chosen": -2.1019277572631836,
      "logits/rejected": -2.0322742462158203,
      "logps/chosen": -245.09219360351562,
      "logps/rejected": -329.4565734863281,
      "loss": 0.0469,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.797667503356934,
      "rewards/margins": 7.920835494995117,
      "rewards/rejected": -12.718502044677734,
      "step": 2181
    },
    {
      "epoch": 2.86,
      "learning_rate": 4.1168773499898683e-05,
      "logits/chosen": -1.9382474422454834,
      "logits/rejected": -1.9113414287567139,
      "logps/chosen": -236.1922149658203,
      "logps/rejected": -353.05841064453125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.7860846519470215,
      "rewards/margins": 10.678539276123047,
      "rewards/rejected": -15.464625358581543,
      "step": 2182
    },
    {
      "epoch": 2.86,
      "learning_rate": 4.1160827478966043e-05,
      "logits/chosen": -2.0668325424194336,
      "logits/rejected": -2.1790409088134766,
      "logps/chosen": -292.84307861328125,
      "logps/rejected": -359.0924987792969,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.625582695007324,
      "rewards/margins": 7.887023448944092,
      "rewards/rejected": -13.512606620788574,
      "step": 2183
    },
    {
      "epoch": 2.86,
      "learning_rate": 4.1152878652466433e-05,
      "logits/chosen": -2.0241265296936035,
      "logits/rejected": -2.0191540718078613,
      "logps/chosen": -287.0768127441406,
      "logps/rejected": -323.9988708496094,
      "loss": 0.0987,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.580556869506836,
      "rewards/margins": 7.032944679260254,
      "rewards/rejected": -12.613500595092773,
      "step": 2184
    },
    {
      "epoch": 2.86,
      "learning_rate": 4.114492702177978e-05,
      "logits/chosen": -2.0294430255889893,
      "logits/rejected": -2.0676746368408203,
      "logps/chosen": -222.878662109375,
      "logps/rejected": -280.4661865234375,
      "loss": 0.0881,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.30618953704834,
      "rewards/margins": 7.54060697555542,
      "rewards/rejected": -11.846796989440918,
      "step": 2185
    },
    {
      "epoch": 2.86,
      "learning_rate": 4.113697258828651e-05,
      "logits/chosen": -2.190463066101074,
      "logits/rejected": -2.068010091781616,
      "logps/chosen": -263.63275146484375,
      "logps/rejected": -285.02313232421875,
      "loss": 0.1216,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.808750152587891,
      "rewards/margins": 6.679539203643799,
      "rewards/rejected": -11.488289833068848,
      "step": 2186
    },
    {
      "epoch": 2.86,
      "learning_rate": 4.112901535336754e-05,
      "logits/chosen": -1.9238157272338867,
      "logits/rejected": -1.92644202709198,
      "logps/chosen": -250.43479919433594,
      "logps/rejected": -306.7027893066406,
      "loss": 0.0481,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.677032470703125,
      "rewards/margins": 7.560239791870117,
      "rewards/rejected": -12.237272262573242,
      "step": 2187
    },
    {
      "epoch": 2.86,
      "learning_rate": 4.1121055318404264e-05,
      "logits/chosen": -1.8979827165603638,
      "logits/rejected": -1.932413935661316,
      "logps/chosen": -185.6468505859375,
      "logps/rejected": -269.4192810058594,
      "loss": 0.131,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.4043474197387695,
      "rewards/margins": 6.746544361114502,
      "rewards/rejected": -11.150891304016113,
      "step": 2188
    },
    {
      "epoch": 2.86,
      "learning_rate": 4.111309248477858e-05,
      "logits/chosen": -2.0596837997436523,
      "logits/rejected": -2.1338276863098145,
      "logps/chosen": -225.3911895751953,
      "logps/rejected": -299.9465026855469,
      "loss": 0.0501,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.601292133331299,
      "rewards/margins": 7.340612888336182,
      "rewards/rejected": -11.941904067993164,
      "step": 2189
    },
    {
      "epoch": 2.87,
      "learning_rate": 4.1105126853872845e-05,
      "logits/chosen": -1.857706069946289,
      "logits/rejected": -1.870910406112671,
      "logps/chosen": -259.96826171875,
      "logps/rejected": -323.454833984375,
      "loss": 0.0719,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.881323337554932,
      "rewards/margins": 7.966041564941406,
      "rewards/rejected": -13.84736442565918,
      "step": 2190
    },
    {
      "epoch": 2.87,
      "learning_rate": 4.109715842706993e-05,
      "logits/chosen": -1.9305803775787354,
      "logits/rejected": -1.949027180671692,
      "logps/chosen": -190.01832580566406,
      "logps/rejected": -272.2169189453125,
      "loss": 0.1018,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.9668006896972656,
      "rewards/margins": 7.031905651092529,
      "rewards/rejected": -10.998706817626953,
      "step": 2191
    },
    {
      "epoch": 2.87,
      "learning_rate": 4.1089187205753164e-05,
      "logits/chosen": -2.0830509662628174,
      "logits/rejected": -2.0943126678466797,
      "logps/chosen": -226.88882446289062,
      "logps/rejected": -289.788330078125,
      "loss": 0.1208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.195775508880615,
      "rewards/margins": 6.762811660766602,
      "rewards/rejected": -10.958586692810059,
      "step": 2192
    },
    {
      "epoch": 2.87,
      "learning_rate": 4.108121319130638e-05,
      "logits/chosen": -1.9216828346252441,
      "logits/rejected": -2.0078539848327637,
      "logps/chosen": -209.600830078125,
      "logps/rejected": -294.1441955566406,
      "loss": 0.0111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5034751892089844,
      "rewards/margins": 8.239934921264648,
      "rewards/rejected": -11.743410110473633,
      "step": 2193
    },
    {
      "epoch": 2.87,
      "learning_rate": 4.107323638511389e-05,
      "logits/chosen": -1.9660532474517822,
      "logits/rejected": -2.1336467266082764,
      "logps/chosen": -212.7504425048828,
      "logps/rejected": -326.82940673828125,
      "loss": 0.0888,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.168840408325195,
      "rewards/margins": 7.612202167510986,
      "rewards/rejected": -11.781042098999023,
      "step": 2194
    },
    {
      "epoch": 2.87,
      "learning_rate": 4.106525678856049e-05,
      "logits/chosen": -2.172245979309082,
      "logits/rejected": -2.1358203887939453,
      "logps/chosen": -276.8701477050781,
      "logps/rejected": -380.2010803222656,
      "loss": 0.021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.663802146911621,
      "rewards/margins": 9.204500198364258,
      "rewards/rejected": -14.868301391601562,
      "step": 2195
    },
    {
      "epoch": 2.87,
      "learning_rate": 4.1057274403031456e-05,
      "logits/chosen": -2.169541120529175,
      "logits/rejected": -2.206789493560791,
      "logps/chosen": -210.1413116455078,
      "logps/rejected": -250.99085998535156,
      "loss": 0.0912,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.160667896270752,
      "rewards/margins": 6.4327392578125,
      "rewards/rejected": -11.593406677246094,
      "step": 2196
    },
    {
      "epoch": 2.88,
      "learning_rate": 4.104928922991256e-05,
      "logits/chosen": -1.8071986436843872,
      "logits/rejected": -1.9281902313232422,
      "logps/chosen": -195.72409057617188,
      "logps/rejected": -309.42364501953125,
      "loss": 0.0709,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.420380592346191,
      "rewards/margins": 7.542280673980713,
      "rewards/rejected": -12.96265983581543,
      "step": 2197
    },
    {
      "epoch": 2.88,
      "learning_rate": 4.104130127059006e-05,
      "logits/chosen": -1.4423290491104126,
      "logits/rejected": -1.486830472946167,
      "logps/chosen": -245.29315185546875,
      "logps/rejected": -349.1470642089844,
      "loss": 0.0448,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.846521854400635,
      "rewards/margins": 8.050249099731445,
      "rewards/rejected": -12.896770477294922,
      "step": 2198
    },
    {
      "epoch": 2.88,
      "learning_rate": 4.1033310526450675e-05,
      "logits/chosen": -2.0059428215026855,
      "logits/rejected": -2.086317777633667,
      "logps/chosen": -246.69886779785156,
      "logps/rejected": -341.550048828125,
      "loss": 0.1013,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.417656898498535,
      "rewards/margins": 7.65479040145874,
      "rewards/rejected": -12.0724458694458,
      "step": 2199
    },
    {
      "epoch": 2.88,
      "learning_rate": 4.102531699888163e-05,
      "logits/chosen": -1.9524719715118408,
      "logits/rejected": -2.0715389251708984,
      "logps/chosen": -200.7295379638672,
      "logps/rejected": -330.2983703613281,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.0946879386901855,
      "rewards/margins": 8.889674186706543,
      "rewards/rejected": -12.984362602233887,
      "step": 2200
    },
    {
      "epoch": 2.88,
      "learning_rate": 4.1017320689270614e-05,
      "logits/chosen": -1.9974534511566162,
      "logits/rejected": -1.9925034046173096,
      "logps/chosen": -245.26065063476562,
      "logps/rejected": -287.6265563964844,
      "loss": 0.094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.502739906311035,
      "rewards/margins": 6.654064655303955,
      "rewards/rejected": -10.156805038452148,
      "step": 2201
    },
    {
      "epoch": 2.88,
      "learning_rate": 4.100932159900582e-05,
      "logits/chosen": -1.9044215679168701,
      "logits/rejected": -1.944065809249878,
      "logps/chosen": -229.14227294921875,
      "logps/rejected": -277.8479919433594,
      "loss": 0.0599,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.378231048583984,
      "rewards/margins": 5.870229721069336,
      "rewards/rejected": -10.24846076965332,
      "step": 2202
    },
    {
      "epoch": 2.88,
      "learning_rate": 4.1001319729475916e-05,
      "logits/chosen": -2.0188095569610596,
      "logits/rejected": -2.0112361907958984,
      "logps/chosen": -214.5211181640625,
      "logps/rejected": -300.25091552734375,
      "loss": 0.0069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.251053333282471,
      "rewards/margins": 8.576237678527832,
      "rewards/rejected": -13.827289581298828,
      "step": 2203
    },
    {
      "epoch": 2.88,
      "learning_rate": 4.099331508207005e-05,
      "logits/chosen": -1.9385230541229248,
      "logits/rejected": -1.9427070617675781,
      "logps/chosen": -234.5819854736328,
      "logps/rejected": -330.6009826660156,
      "loss": 0.1316,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.050275802612305,
      "rewards/margins": 7.512202739715576,
      "rewards/rejected": -12.562477111816406,
      "step": 2204
    },
    {
      "epoch": 2.89,
      "learning_rate": 4.098530765817784e-05,
      "logits/chosen": -2.026866912841797,
      "logits/rejected": -2.1532793045043945,
      "logps/chosen": -199.71560668945312,
      "logps/rejected": -320.5267028808594,
      "loss": 0.0092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.68998908996582,
      "rewards/margins": 7.659614562988281,
      "rewards/rejected": -12.349604606628418,
      "step": 2205
    },
    {
      "epoch": 2.89,
      "learning_rate": 4.09772974591894e-05,
      "logits/chosen": -1.7982935905456543,
      "logits/rejected": -1.8411973714828491,
      "logps/chosen": -237.09228515625,
      "logps/rejected": -322.3974609375,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.479815483093262,
      "rewards/margins": 8.23798656463623,
      "rewards/rejected": -13.717802047729492,
      "step": 2206
    },
    {
      "epoch": 2.89,
      "learning_rate": 4.0969284486495335e-05,
      "logits/chosen": -1.76961088180542,
      "logits/rejected": -1.8508358001708984,
      "logps/chosen": -240.76266479492188,
      "logps/rejected": -342.0540771484375,
      "loss": 0.044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.3279876708984375,
      "rewards/margins": 9.263980865478516,
      "rewards/rejected": -14.591968536376953,
      "step": 2207
    },
    {
      "epoch": 2.89,
      "learning_rate": 4.0961268741486716e-05,
      "logits/chosen": -1.981732726097107,
      "logits/rejected": -2.117330551147461,
      "logps/chosen": -237.18115234375,
      "logps/rejected": -341.62762451171875,
      "loss": 0.0871,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.773239612579346,
      "rewards/margins": 8.926799774169922,
      "rewards/rejected": -14.700039863586426,
      "step": 2208
    },
    {
      "epoch": 2.89,
      "learning_rate": 4.09532502255551e-05,
      "logits/chosen": -2.046422004699707,
      "logits/rejected": -2.099958658218384,
      "logps/chosen": -232.92481994628906,
      "logps/rejected": -325.4584655761719,
      "loss": 0.0145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.9477105140686035,
      "rewards/margins": 8.676502227783203,
      "rewards/rejected": -13.624212265014648,
      "step": 2209
    },
    {
      "epoch": 2.89,
      "learning_rate": 4.094522894009251e-05,
      "logits/chosen": -1.593595027923584,
      "logits/rejected": -1.5732452869415283,
      "logps/chosen": -209.82452392578125,
      "logps/rejected": -287.2315673828125,
      "loss": 0.0061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.161198139190674,
      "rewards/margins": 8.423956871032715,
      "rewards/rejected": -12.585155487060547,
      "step": 2210
    },
    {
      "epoch": 2.89,
      "learning_rate": 4.0937204886491485e-05,
      "logits/chosen": -1.9955334663391113,
      "logits/rejected": -2.0291831493377686,
      "logps/chosen": -206.82830810546875,
      "logps/rejected": -286.96966552734375,
      "loss": 0.0463,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.212306022644043,
      "rewards/margins": 7.9994988441467285,
      "rewards/rejected": -13.211803436279297,
      "step": 2211
    },
    {
      "epoch": 2.89,
      "learning_rate": 4.092917806614503e-05,
      "logits/chosen": -2.0906317234039307,
      "logits/rejected": -2.156611442565918,
      "logps/chosen": -244.69973754882812,
      "logps/rejected": -351.0030822753906,
      "loss": 0.0988,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.618649005889893,
      "rewards/margins": 8.488447189331055,
      "rewards/rejected": -14.107095718383789,
      "step": 2212
    },
    {
      "epoch": 2.9,
      "learning_rate": 4.092114848044659e-05,
      "logits/chosen": -1.9830385446548462,
      "logits/rejected": -2.0258727073669434,
      "logps/chosen": -236.79864501953125,
      "logps/rejected": -335.24591064453125,
      "loss": 0.0021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.998775959014893,
      "rewards/margins": 10.42489242553711,
      "rewards/rejected": -15.423666954040527,
      "step": 2213
    },
    {
      "epoch": 2.9,
      "learning_rate": 4.0913116130790164e-05,
      "logits/chosen": -2.1110739707946777,
      "logits/rejected": -2.1509482860565186,
      "logps/chosen": -217.78807067871094,
      "logps/rejected": -330.7835388183594,
      "loss": 0.0903,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.235716819763184,
      "rewards/margins": 7.777309417724609,
      "rewards/rejected": -13.01302719116211,
      "step": 2214
    },
    {
      "epoch": 2.9,
      "learning_rate": 4.090508101857017e-05,
      "logits/chosen": -2.1012043952941895,
      "logits/rejected": -2.162986993789673,
      "logps/chosen": -242.8408660888672,
      "logps/rejected": -334.86444091796875,
      "loss": 0.0873,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.359984397888184,
      "rewards/margins": 9.122700691223145,
      "rewards/rejected": -13.482685089111328,
      "step": 2215
    },
    {
      "epoch": 2.9,
      "learning_rate": 4.089704314518152e-05,
      "logits/chosen": -1.97257661819458,
      "logits/rejected": -2.0269057750701904,
      "logps/chosen": -178.48583984375,
      "logps/rejected": -294.429931640625,
      "loss": 0.1325,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.222825050354004,
      "rewards/margins": 8.763550758361816,
      "rewards/rejected": -12.98637580871582,
      "step": 2216
    },
    {
      "epoch": 2.9,
      "learning_rate": 4.088900251201964e-05,
      "logits/chosen": -2.030012607574463,
      "logits/rejected": -1.9918521642684937,
      "logps/chosen": -221.71617126464844,
      "logps/rejected": -312.90399169921875,
      "loss": 0.0141,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6067910194396973,
      "rewards/margins": 9.348206520080566,
      "rewards/rejected": -12.954997062683105,
      "step": 2217
    },
    {
      "epoch": 2.9,
      "learning_rate": 4.088095912048038e-05,
      "logits/chosen": -2.0294079780578613,
      "logits/rejected": -2.2200558185577393,
      "logps/chosen": -195.28195190429688,
      "logps/rejected": -322.5656433105469,
      "loss": 0.0878,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.590506553649902,
      "rewards/margins": 7.0590667724609375,
      "rewards/rejected": -11.649572372436523,
      "step": 2218
    },
    {
      "epoch": 2.9,
      "learning_rate": 4.08729129719601e-05,
      "logits/chosen": -2.005953550338745,
      "logits/rejected": -2.0370614528656006,
      "logps/chosen": -192.7057647705078,
      "logps/rejected": -234.94427490234375,
      "loss": 0.1856,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.630458354949951,
      "rewards/margins": 5.305692195892334,
      "rewards/rejected": -9.936150550842285,
      "step": 2219
    },
    {
      "epoch": 2.91,
      "learning_rate": 4.086486406785566e-05,
      "logits/chosen": -2.1289520263671875,
      "logits/rejected": -2.2230443954467773,
      "logps/chosen": -262.62725830078125,
      "logps/rejected": -385.7099609375,
      "loss": 0.1385,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.901250839233398,
      "rewards/margins": 6.619694232940674,
      "rewards/rejected": -11.52094554901123,
      "step": 2220
    },
    {
      "epoch": 2.91,
      "learning_rate": 4.085681240956434e-05,
      "logits/chosen": -2.0626068115234375,
      "logits/rejected": -2.1245715618133545,
      "logps/chosen": -244.63568115234375,
      "logps/rejected": -400.87127685546875,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.728460311889648,
      "rewards/margins": 10.4347562789917,
      "rewards/rejected": -15.163215637207031,
      "step": 2221
    },
    {
      "epoch": 2.91,
      "learning_rate": 4.084875799848395e-05,
      "logits/chosen": -1.9657922983169556,
      "logits/rejected": -2.028837203979492,
      "logps/chosen": -246.70101928710938,
      "logps/rejected": -343.2515563964844,
      "loss": 0.0514,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.407756805419922,
      "rewards/margins": 7.938384532928467,
      "rewards/rejected": -12.346141815185547,
      "step": 2222
    },
    {
      "epoch": 2.91,
      "learning_rate": 4.084070083601276e-05,
      "logits/chosen": -2.0747859477996826,
      "logits/rejected": -1.979410171508789,
      "logps/chosen": -273.43145751953125,
      "logps/rejected": -331.90582275390625,
      "loss": 0.0018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.8637495040893555,
      "rewards/margins": 8.859865188598633,
      "rewards/rejected": -14.723615646362305,
      "step": 2223
    },
    {
      "epoch": 2.91,
      "learning_rate": 4.0832640923549515e-05,
      "logits/chosen": -1.9947311878204346,
      "logits/rejected": -2.0264732837677,
      "logps/chosen": -225.6526336669922,
      "logps/rejected": -317.21722412109375,
      "loss": 0.0493,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.485606670379639,
      "rewards/margins": 8.677902221679688,
      "rewards/rejected": -14.1635103225708,
      "step": 2224
    },
    {
      "epoch": 2.91,
      "learning_rate": 4.082457826249343e-05,
      "logits/chosen": -2.0638132095336914,
      "logits/rejected": -2.0210137367248535,
      "logps/chosen": -264.2833251953125,
      "logps/rejected": -337.869140625,
      "loss": 0.0891,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.095065593719482,
      "rewards/margins": 7.04831075668335,
      "rewards/rejected": -12.143376350402832,
      "step": 2225
    },
    {
      "epoch": 2.91,
      "learning_rate": 4.0816512854244224e-05,
      "logits/chosen": -1.7565228939056396,
      "logits/rejected": -1.877976894378662,
      "logps/chosen": -218.58175659179688,
      "logps/rejected": -353.1058654785156,
      "loss": 0.0879,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.139636993408203,
      "rewards/margins": 9.21428108215332,
      "rewards/rejected": -13.353917121887207,
      "step": 2226
    },
    {
      "epoch": 2.91,
      "learning_rate": 4.080844470020205e-05,
      "logits/chosen": -2.0760161876678467,
      "logits/rejected": -2.198225259780884,
      "logps/chosen": -237.87025451660156,
      "logps/rejected": -352.52825927734375,
      "loss": 0.0054,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.9761271476745605,
      "rewards/margins": 8.944655418395996,
      "rewards/rejected": -13.920783042907715,
      "step": 2227
    },
    {
      "epoch": 2.92,
      "learning_rate": 4.08003738017676e-05,
      "logits/chosen": -2.160156488418579,
      "logits/rejected": -2.2180447578430176,
      "logps/chosen": -267.6474304199219,
      "logps/rejected": -358.6483459472656,
      "loss": 0.009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.819602012634277,
      "rewards/margins": 7.647256851196289,
      "rewards/rejected": -13.466859817504883,
      "step": 2228
    },
    {
      "epoch": 2.92,
      "learning_rate": 4.079230016034197e-05,
      "logits/chosen": -1.7095756530761719,
      "logits/rejected": -1.7194485664367676,
      "logps/chosen": -226.6480712890625,
      "logps/rejected": -303.1681213378906,
      "loss": 0.0772,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.34532356262207,
      "rewards/margins": 7.812335968017578,
      "rewards/rejected": -12.157660484313965,
      "step": 2229
    },
    {
      "epoch": 2.92,
      "learning_rate": 4.078422377732678e-05,
      "logits/chosen": -2.097193956375122,
      "logits/rejected": -2.124293327331543,
      "logps/chosen": -249.19630432128906,
      "logps/rejected": -346.5874938964844,
      "loss": 0.0541,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.2485575675964355,
      "rewards/margins": 7.850658893585205,
      "rewards/rejected": -13.09921646118164,
      "step": 2230
    },
    {
      "epoch": 2.92,
      "learning_rate": 4.0776144654124125e-05,
      "logits/chosen": -2.2765426635742188,
      "logits/rejected": -2.270753860473633,
      "logps/chosen": -241.84669494628906,
      "logps/rejected": -336.55096435546875,
      "loss": 0.0791,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.211452007293701,
      "rewards/margins": 8.208662033081055,
      "rewards/rejected": -14.420112609863281,
      "step": 2231
    },
    {
      "epoch": 2.92,
      "learning_rate": 4.0768062792136555e-05,
      "logits/chosen": -2.1474051475524902,
      "logits/rejected": -2.1365041732788086,
      "logps/chosen": -244.55490112304688,
      "logps/rejected": -346.3648376464844,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.497187614440918,
      "rewards/margins": 8.268601417541504,
      "rewards/rejected": -12.765789031982422,
      "step": 2232
    },
    {
      "epoch": 2.92,
      "learning_rate": 4.07599781927671e-05,
      "logits/chosen": -2.1392455101013184,
      "logits/rejected": -2.142500400543213,
      "logps/chosen": -250.07745361328125,
      "logps/rejected": -311.97808837890625,
      "loss": 0.1738,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.954948425292969,
      "rewards/margins": 6.751895427703857,
      "rewards/rejected": -11.706844329833984,
      "step": 2233
    },
    {
      "epoch": 2.92,
      "learning_rate": 4.075189085741927e-05,
      "logits/chosen": -2.1242191791534424,
      "logits/rejected": -2.1828739643096924,
      "logps/chosen": -265.0927429199219,
      "logps/rejected": -344.9898376464844,
      "loss": 0.0035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.8403639793396,
      "rewards/margins": 8.913230895996094,
      "rewards/rejected": -14.753594398498535,
      "step": 2234
    },
    {
      "epoch": 2.92,
      "learning_rate": 4.074380078749707e-05,
      "logits/chosen": -1.9724620580673218,
      "logits/rejected": -2.0979995727539062,
      "logps/chosen": -230.10606384277344,
      "logps/rejected": -340.49420166015625,
      "loss": 0.0882,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.136164665222168,
      "rewards/margins": 7.166202068328857,
      "rewards/rejected": -12.302366256713867,
      "step": 2235
    },
    {
      "epoch": 2.93,
      "learning_rate": 4.073570798440493e-05,
      "logits/chosen": -2.173654794692993,
      "logits/rejected": -2.158339262008667,
      "logps/chosen": -264.9515075683594,
      "logps/rejected": -385.02947998046875,
      "loss": 0.0441,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.116758823394775,
      "rewards/margins": 10.693913459777832,
      "rewards/rejected": -14.810672760009766,
      "step": 2236
    },
    {
      "epoch": 2.93,
      "learning_rate": 4.072761244954781e-05,
      "logits/chosen": -1.9193484783172607,
      "logits/rejected": -1.9324231147766113,
      "logps/chosen": -229.14004516601562,
      "logps/rejected": -309.6470031738281,
      "loss": 0.0882,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.022686004638672,
      "rewards/margins": 7.921196460723877,
      "rewards/rejected": -13.94388198852539,
      "step": 2237
    },
    {
      "epoch": 2.93,
      "learning_rate": 4.07195141843311e-05,
      "logits/chosen": -2.1049351692199707,
      "logits/rejected": -2.204519271850586,
      "logps/chosen": -237.9530487060547,
      "logps/rejected": -337.65203857421875,
      "loss": 0.0497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.841470718383789,
      "rewards/margins": 7.777152061462402,
      "rewards/rejected": -13.618623733520508,
      "step": 2238
    },
    {
      "epoch": 2.93,
      "learning_rate": 4.07114131901607e-05,
      "logits/chosen": -1.9116312265396118,
      "logits/rejected": -2.049605131149292,
      "logps/chosen": -310.1016845703125,
      "logps/rejected": -415.382568359375,
      "loss": 0.0878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.793592929840088,
      "rewards/margins": 7.996792793273926,
      "rewards/rejected": -13.790386199951172,
      "step": 2239
    },
    {
      "epoch": 2.93,
      "learning_rate": 4.070330946844295e-05,
      "logits/chosen": -2.2676711082458496,
      "logits/rejected": -2.268871545791626,
      "logps/chosen": -257.8155212402344,
      "logps/rejected": -305.8218688964844,
      "loss": 0.0539,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.3989386558532715,
      "rewards/margins": 7.469062805175781,
      "rewards/rejected": -12.868001937866211,
      "step": 2240
    },
    {
      "epoch": 2.93,
      "learning_rate": 4.069520302058469e-05,
      "logits/chosen": -2.0371265411376953,
      "logits/rejected": -2.0505199432373047,
      "logps/chosen": -187.00189208984375,
      "logps/rejected": -225.2563934326172,
      "loss": 0.1371,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.377021789550781,
      "rewards/margins": 5.380834579467773,
      "rewards/rejected": -9.757856369018555,
      "step": 2241
    },
    {
      "epoch": 2.93,
      "learning_rate": 4.068709384799322e-05,
      "logits/chosen": -2.2195491790771484,
      "logits/rejected": -2.273913621902466,
      "logps/chosen": -231.99951171875,
      "logps/rejected": -323.251953125,
      "loss": 0.0899,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.185029983520508,
      "rewards/margins": 8.207600593566895,
      "rewards/rejected": -13.392631530761719,
      "step": 2242
    },
    {
      "epoch": 2.94,
      "learning_rate": 4.067898195207631e-05,
      "logits/chosen": -1.9835385084152222,
      "logits/rejected": -2.0143542289733887,
      "logps/chosen": -260.90447998046875,
      "logps/rejected": -374.3984680175781,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.085165977478027,
      "rewards/margins": 9.697397232055664,
      "rewards/rejected": -14.782563209533691,
      "step": 2243
    },
    {
      "epoch": 2.94,
      "learning_rate": 4.0670867334242223e-05,
      "logits/chosen": -2.0046844482421875,
      "logits/rejected": -2.1477673053741455,
      "logps/chosen": -255.0173797607422,
      "logps/rejected": -383.2938537597656,
      "loss": 0.0578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.924704074859619,
      "rewards/margins": 8.287879943847656,
      "rewards/rejected": -14.212583541870117,
      "step": 2244
    },
    {
      "epoch": 2.94,
      "learning_rate": 4.0662749995899666e-05,
      "logits/chosen": -1.951084017753601,
      "logits/rejected": -2.0860135555267334,
      "logps/chosen": -234.13211059570312,
      "logps/rejected": -325.2522888183594,
      "loss": 0.044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.4876708984375,
      "rewards/margins": 8.078499794006348,
      "rewards/rejected": -14.566171646118164,
      "step": 2245
    },
    {
      "epoch": 2.94,
      "learning_rate": 4.065462993845784e-05,
      "logits/chosen": -2.1116063594818115,
      "logits/rejected": -2.1521196365356445,
      "logps/chosen": -194.48851013183594,
      "logps/rejected": -307.51019287109375,
      "loss": 0.131,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.6868486404418945,
      "rewards/margins": 7.20567512512207,
      "rewards/rejected": -11.892522811889648,
      "step": 2246
    },
    {
      "epoch": 2.94,
      "learning_rate": 4.064650716332642e-05,
      "logits/chosen": -2.218405246734619,
      "logits/rejected": -2.277463436126709,
      "logps/chosen": -212.8626251220703,
      "logps/rejected": -265.5347595214844,
      "loss": 0.0454,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.655837535858154,
      "rewards/margins": 6.375062942504883,
      "rewards/rejected": -12.030900955200195,
      "step": 2247
    },
    {
      "epoch": 2.94,
      "learning_rate": 4.063838167191553e-05,
      "logits/chosen": -2.099470615386963,
      "logits/rejected": -2.1984596252441406,
      "logps/chosen": -228.62374877929688,
      "logps/rejected": -336.2490539550781,
      "loss": 0.0444,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.770306587219238,
      "rewards/margins": 8.472064971923828,
      "rewards/rejected": -14.2423734664917,
      "step": 2248
    },
    {
      "epoch": 2.94,
      "learning_rate": 4.063025346563577e-05,
      "logits/chosen": -2.292811393737793,
      "logits/rejected": -2.297818660736084,
      "logps/chosen": -244.33920288085938,
      "logps/rejected": -344.06219482421875,
      "loss": 0.0031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.461674690246582,
      "rewards/margins": 8.859297752380371,
      "rewards/rejected": -14.320972442626953,
      "step": 2249
    },
    {
      "epoch": 2.94,
      "learning_rate": 4.0622122545898246e-05,
      "logits/chosen": -1.9312044382095337,
      "logits/rejected": -1.9553238153457642,
      "logps/chosen": -278.5567932128906,
      "logps/rejected": -343.84344482421875,
      "loss": 0.0872,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.784635066986084,
      "rewards/margins": 7.91162109375,
      "rewards/rejected": -13.696255683898926,
      "step": 2250
    },
    {
      "epoch": 2.95,
      "learning_rate": 4.061398891411449e-05,
      "logits/chosen": -2.0505380630493164,
      "logits/rejected": -2.088383913040161,
      "logps/chosen": -240.8919677734375,
      "logps/rejected": -322.6705322265625,
      "loss": 0.0475,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.79270076751709,
      "rewards/margins": 7.686241626739502,
      "rewards/rejected": -13.478941917419434,
      "step": 2251
    },
    {
      "epoch": 2.95,
      "learning_rate": 4.060585257169653e-05,
      "logits/chosen": -2.134075880050659,
      "logits/rejected": -2.1802515983581543,
      "logps/chosen": -235.2249755859375,
      "logps/rejected": -284.25543212890625,
      "loss": 0.0896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.834575176239014,
      "rewards/margins": 7.375324249267578,
      "rewards/rejected": -12.209898948669434,
      "step": 2252
    },
    {
      "epoch": 2.95,
      "learning_rate": 4.059771352005686e-05,
      "logits/chosen": -2.306246757507324,
      "logits/rejected": -2.350177764892578,
      "logps/chosen": -266.5599060058594,
      "logps/rejected": -339.7952880859375,
      "loss": 0.0466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.026610851287842,
      "rewards/margins": 8.363250732421875,
      "rewards/rejected": -13.389862060546875,
      "step": 2253
    },
    {
      "epoch": 2.95,
      "learning_rate": 4.0589571760608443e-05,
      "logits/chosen": -1.755591869354248,
      "logits/rejected": -1.799825668334961,
      "logps/chosen": -215.43942260742188,
      "logps/rejected": -324.50396728515625,
      "loss": 0.095,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.952765941619873,
      "rewards/margins": 9.143765449523926,
      "rewards/rejected": -14.096531867980957,
      "step": 2254
    },
    {
      "epoch": 2.95,
      "learning_rate": 4.058142729476471e-05,
      "logits/chosen": -2.1598591804504395,
      "logits/rejected": -2.207771062850952,
      "logps/chosen": -218.27322387695312,
      "logps/rejected": -281.4322509765625,
      "loss": 0.0652,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.535760879516602,
      "rewards/margins": 6.41301155090332,
      "rewards/rejected": -12.948771476745605,
      "step": 2255
    },
    {
      "epoch": 2.95,
      "learning_rate": 4.0573280123939574e-05,
      "logits/chosen": -2.2826626300811768,
      "logits/rejected": -2.242478609085083,
      "logps/chosen": -225.7889404296875,
      "logps/rejected": -294.6713562011719,
      "loss": 0.1417,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.481799602508545,
      "rewards/margins": 7.591139793395996,
      "rewards/rejected": -12.0729398727417,
      "step": 2256
    },
    {
      "epoch": 2.95,
      "learning_rate": 4.0565130249547396e-05,
      "logits/chosen": -2.0708048343658447,
      "logits/rejected": -2.065812110900879,
      "logps/chosen": -216.64683532714844,
      "logps/rejected": -314.45892333984375,
      "loss": 0.0447,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.770445346832275,
      "rewards/margins": 8.61684799194336,
      "rewards/rejected": -13.387293815612793,
      "step": 2257
    },
    {
      "epoch": 2.96,
      "learning_rate": 4.055697767300302e-05,
      "logits/chosen": -2.217392683029175,
      "logits/rejected": -2.2642881870269775,
      "logps/chosen": -229.09271240234375,
      "logps/rejected": -312.7999572753906,
      "loss": 0.1442,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.301823139190674,
      "rewards/margins": 7.752106666564941,
      "rewards/rejected": -12.053930282592773,
      "step": 2258
    },
    {
      "epoch": 2.96,
      "learning_rate": 4.054882239572176e-05,
      "logits/chosen": -1.9833722114562988,
      "logits/rejected": -2.1198909282684326,
      "logps/chosen": -219.9600067138672,
      "logps/rejected": -289.3666687011719,
      "loss": 0.197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.9837100505828857,
      "rewards/margins": 9.081433296203613,
      "rewards/rejected": -12.065143585205078,
      "step": 2259
    },
    {
      "epoch": 2.96,
      "learning_rate": 4.054066441911939e-05,
      "logits/chosen": -2.0739023685455322,
      "logits/rejected": -2.0905725955963135,
      "logps/chosen": -217.68226623535156,
      "logps/rejected": -290.0678405761719,
      "loss": 0.1176,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.164804935455322,
      "rewards/margins": 6.717545986175537,
      "rewards/rejected": -11.882349967956543,
      "step": 2260
    },
    {
      "epoch": 2.96,
      "learning_rate": 4.053250374461217e-05,
      "logits/chosen": -2.0488510131835938,
      "logits/rejected": -2.0801382064819336,
      "logps/chosen": -197.89596557617188,
      "logps/rejected": -301.1357727050781,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.089158535003662,
      "rewards/margins": 8.928377151489258,
      "rewards/rejected": -13.017536163330078,
      "step": 2261
    },
    {
      "epoch": 2.96,
      "learning_rate": 4.052434037361681e-05,
      "logits/chosen": -2.1752076148986816,
      "logits/rejected": -2.2135376930236816,
      "logps/chosen": -196.59396362304688,
      "logps/rejected": -283.71185302734375,
      "loss": 0.0888,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.9845967292785645,
      "rewards/margins": 6.94215202331543,
      "rewards/rejected": -11.926748275756836,
      "step": 2262
    },
    {
      "epoch": 2.96,
      "learning_rate": 4.05161743075505e-05,
      "logits/chosen": -1.9602102041244507,
      "logits/rejected": -1.9223976135253906,
      "logps/chosen": -233.06593322753906,
      "logps/rejected": -308.6895751953125,
      "loss": 0.0482,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.860264301300049,
      "rewards/margins": 8.868692398071289,
      "rewards/rejected": -13.72895622253418,
      "step": 2263
    },
    {
      "epoch": 2.96,
      "learning_rate": 4.0508005547830884e-05,
      "logits/chosen": -1.8972437381744385,
      "logits/rejected": -2.00484561920166,
      "logps/chosen": -246.32203674316406,
      "logps/rejected": -347.1911926269531,
      "loss": 0.0046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.049254894256592,
      "rewards/margins": 9.062969207763672,
      "rewards/rejected": -14.112222671508789,
      "step": 2264
    },
    {
      "epoch": 2.96,
      "learning_rate": 4.049983409587608e-05,
      "logits/chosen": -1.9605618715286255,
      "logits/rejected": -1.8788563013076782,
      "logps/chosen": -222.1049041748047,
      "logps/rejected": -309.7162170410156,
      "loss": 0.0104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.519625663757324,
      "rewards/margins": 8.124978065490723,
      "rewards/rejected": -13.644603729248047,
      "step": 2265
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.049165995310469e-05,
      "logits/chosen": -1.9594004154205322,
      "logits/rejected": -1.9734030961990356,
      "logps/chosen": -238.72772216796875,
      "logps/rejected": -329.5353698730469,
      "loss": 0.0983,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.496771812438965,
      "rewards/margins": 7.7921142578125,
      "rewards/rejected": -13.288885116577148,
      "step": 2266
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.0483483120935766e-05,
      "logits/chosen": -1.8387665748596191,
      "logits/rejected": -1.8579622507095337,
      "logps/chosen": -244.2477569580078,
      "logps/rejected": -308.0921325683594,
      "loss": 0.0678,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.856629848480225,
      "rewards/margins": 7.74281644821167,
      "rewards/rejected": -12.599447250366211,
      "step": 2267
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.0475303600788825e-05,
      "logits/chosen": -2.012568712234497,
      "logits/rejected": -2.0805506706237793,
      "logps/chosen": -283.11785888671875,
      "logps/rejected": -378.9236755371094,
      "loss": 0.0938,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.643699645996094,
      "rewards/margins": 8.028934478759766,
      "rewards/rejected": -12.67263412475586,
      "step": 2268
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.0467121394083856e-05,
      "logits/chosen": -2.284359931945801,
      "logits/rejected": -2.301757574081421,
      "logps/chosen": -254.15252685546875,
      "logps/rejected": -321.6610412597656,
      "loss": 0.1873,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.626988887786865,
      "rewards/margins": 8.284836769104004,
      "rewards/rejected": -12.911825180053711,
      "step": 2269
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.045893650224132e-05,
      "logits/chosen": -2.092942237854004,
      "logits/rejected": -2.1142632961273193,
      "logps/chosen": -243.60618591308594,
      "logps/rejected": -353.5018005371094,
      "loss": 0.0064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.826841831207275,
      "rewards/margins": 8.407801628112793,
      "rewards/rejected": -13.23464298248291,
      "step": 2270
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.0450748926682125e-05,
      "logits/chosen": -1.9782465696334839,
      "logits/rejected": -2.1163859367370605,
      "logps/chosen": -222.07089233398438,
      "logps/rejected": -366.1741943359375,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.192220211029053,
      "rewards/margins": 8.56313705444336,
      "rewards/rejected": -13.75535774230957,
      "step": 2271
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.044255866882767e-05,
      "logits/chosen": -2.07340669631958,
      "logits/rejected": -2.100149154663086,
      "logps/chosen": -299.82244873046875,
      "logps/rejected": -415.00872802734375,
      "loss": 0.0879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.470133304595947,
      "rewards/margins": 7.713799476623535,
      "rewards/rejected": -12.18393325805664,
      "step": 2272
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.04343657300998e-05,
      "logits/chosen": -2.0520966053009033,
      "logits/rejected": -2.1131479740142822,
      "logps/chosen": -200.32757568359375,
      "logps/rejected": -331.4617614746094,
      "loss": 0.0499,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.1466875076293945,
      "rewards/margins": 9.14694881439209,
      "rewards/rejected": -13.293636322021484,
      "step": 2273
    },
    {
      "epoch": 2.98,
      "learning_rate": 4.042617011192085e-05,
      "logits/chosen": -2.280841112136841,
      "logits/rejected": -2.2835123538970947,
      "logps/chosen": -237.56654357910156,
      "logps/rejected": -305.79193115234375,
      "loss": 0.1117,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.9321794509887695,
      "rewards/margins": 6.701276779174805,
      "rewards/rejected": -11.633456230163574,
      "step": 2274
    },
    {
      "epoch": 2.98,
      "learning_rate": 4.0417971815713584e-05,
      "logits/chosen": -2.2043027877807617,
      "logits/rejected": -2.2083544731140137,
      "logps/chosen": -245.98263549804688,
      "logps/rejected": -355.17608642578125,
      "loss": 0.0449,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.231986999511719,
      "rewards/margins": 8.83634090423584,
      "rewards/rejected": -14.068328857421875,
      "step": 2275
    },
    {
      "epoch": 2.98,
      "learning_rate": 4.040977084290125e-05,
      "logits/chosen": -2.291473627090454,
      "logits/rejected": -2.2201426029205322,
      "logps/chosen": -238.4390106201172,
      "logps/rejected": -292.26910400390625,
      "loss": 0.0063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.370654582977295,
      "rewards/margins": 7.422157287597656,
      "rewards/rejected": -11.792811393737793,
      "step": 2276
    },
    {
      "epoch": 2.98,
      "learning_rate": 4.040156719490757e-05,
      "logits/chosen": -2.1759634017944336,
      "logits/rejected": -2.283318042755127,
      "logps/chosen": -215.19366455078125,
      "logps/rejected": -313.5849914550781,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.866866111755371,
      "rewards/margins": 8.510933876037598,
      "rewards/rejected": -12.377800941467285,
      "step": 2277
    },
    {
      "epoch": 2.98,
      "learning_rate": 4.0393360873156727e-05,
      "logits/chosen": -1.8939836025238037,
      "logits/rejected": -1.9971206188201904,
      "logps/chosen": -264.5501708984375,
      "logps/rejected": -374.10809326171875,
      "loss": 0.1771,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.189629554748535,
      "rewards/margins": 8.558857917785645,
      "rewards/rejected": -12.74848747253418,
      "step": 2278
    },
    {
      "epoch": 2.98,
      "learning_rate": 4.038515187907335e-05,
      "logits/chosen": -2.284393787384033,
      "logits/rejected": -2.287358283996582,
      "logps/chosen": -299.2924499511719,
      "logps/rejected": -349.8957824707031,
      "loss": 0.0971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.67432975769043,
      "rewards/margins": 7.653830528259277,
      "rewards/rejected": -12.328161239624023,
      "step": 2279
    },
    {
      "epoch": 2.98,
      "learning_rate": 4.037694021408255e-05,
      "logits/chosen": -2.0360541343688965,
      "logits/rejected": -2.105217933654785,
      "logps/chosen": -187.9983367919922,
      "logps/rejected": -275.1484069824219,
      "loss": 0.061,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.26163387298584,
      "rewards/margins": 7.959683418273926,
      "rewards/rejected": -12.221317291259766,
      "step": 2280
    },
    {
      "epoch": 2.99,
      "learning_rate": 4.0368725879609896e-05,
      "logits/chosen": -1.9243905544281006,
      "logits/rejected": -1.938022494316101,
      "logps/chosen": -198.75108337402344,
      "logps/rejected": -273.3779296875,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.776736259460449,
      "rewards/margins": 7.805794715881348,
      "rewards/rejected": -11.582530975341797,
      "step": 2281
    },
    {
      "epoch": 2.99,
      "learning_rate": 4.036050887708142e-05,
      "logits/chosen": -2.002272605895996,
      "logits/rejected": -2.020458936691284,
      "logps/chosen": -238.6460723876953,
      "logps/rejected": -387.5722961425781,
      "loss": 0.0868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.070218086242676,
      "rewards/margins": 9.112555503845215,
      "rewards/rejected": -13.18277359008789,
      "step": 2282
    },
    {
      "epoch": 2.99,
      "learning_rate": 4.035228920792361e-05,
      "logits/chosen": -2.1598801612854004,
      "logits/rejected": -2.286231517791748,
      "logps/chosen": -236.18081665039062,
      "logps/rejected": -353.8681640625,
      "loss": 0.0092,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.896045684814453,
      "rewards/margins": 8.549945831298828,
      "rewards/rejected": -13.445991516113281,
      "step": 2283
    },
    {
      "epoch": 2.99,
      "learning_rate": 4.034406687356344e-05,
      "logits/chosen": -2.0794379711151123,
      "logits/rejected": -2.1407833099365234,
      "logps/chosen": -217.3106689453125,
      "logps/rejected": -331.027099609375,
      "loss": 0.0021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.971529006958008,
      "rewards/margins": 9.772381782531738,
      "rewards/rejected": -13.743910789489746,
      "step": 2284
    },
    {
      "epoch": 2.99,
      "learning_rate": 4.0335841875428324e-05,
      "logits/chosen": -1.941980004310608,
      "logits/rejected": -2.0354809761047363,
      "logps/chosen": -201.304443359375,
      "logps/rejected": -340.4508361816406,
      "loss": 0.0923,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.282627582550049,
      "rewards/margins": 7.455456256866455,
      "rewards/rejected": -11.738083839416504,
      "step": 2285
    },
    {
      "epoch": 2.99,
      "learning_rate": 4.032761421494614e-05,
      "logits/chosen": -2.0981268882751465,
      "logits/rejected": -2.087101697921753,
      "logps/chosen": -245.1864471435547,
      "logps/rejected": -277.5719299316406,
      "loss": 0.222,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.013238906860352,
      "rewards/margins": 5.27143669128418,
      "rewards/rejected": -10.284676551818848,
      "step": 2286
    },
    {
      "epoch": 2.99,
      "learning_rate": 4.031938389354524e-05,
      "logits/chosen": -2.0440993309020996,
      "logits/rejected": -2.119518280029297,
      "logps/chosen": -225.06170654296875,
      "logps/rejected": -327.6748352050781,
      "loss": 0.0893,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.085385799407959,
      "rewards/margins": 8.826796531677246,
      "rewards/rejected": -12.912181854248047,
      "step": 2287
    },
    {
      "epoch": 2.99,
      "learning_rate": 4.031115091265444e-05,
      "logits/chosen": -2.212399482727051,
      "logits/rejected": -2.1920952796936035,
      "logps/chosen": -297.997802734375,
      "logps/rejected": -364.1531677246094,
      "loss": 0.0443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.721320152282715,
      "rewards/margins": 7.526367664337158,
      "rewards/rejected": -12.247687339782715,
      "step": 2288
    },
    {
      "epoch": 3.0,
      "learning_rate": 4.030291527370299e-05,
      "logits/chosen": -2.1944549083709717,
      "logits/rejected": -2.1947126388549805,
      "logps/chosen": -259.0519104003906,
      "logps/rejected": -326.65972900390625,
      "loss": 0.0211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.198507308959961,
      "rewards/margins": 8.169037818908691,
      "rewards/rejected": -13.367544174194336,
      "step": 2289
    },
    {
      "epoch": 3.0,
      "learning_rate": 4.029467697812064e-05,
      "logits/chosen": -2.0671141147613525,
      "logits/rejected": -2.0241549015045166,
      "logps/chosen": -231.8189697265625,
      "logps/rejected": -333.82305908203125,
      "loss": 0.0455,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.156424045562744,
      "rewards/margins": 9.836962699890137,
      "rewards/rejected": -13.993387222290039,
      "step": 2290
    },
    {
      "epoch": 3.0,
      "learning_rate": 4.028643602733756e-05,
      "logits/chosen": -1.989417314529419,
      "logits/rejected": -2.149822473526001,
      "logps/chosen": -204.5302276611328,
      "logps/rejected": -314.23602294921875,
      "loss": 0.1316,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.630185604095459,
      "rewards/margins": 9.480209350585938,
      "rewards/rejected": -14.110394477844238,
      "step": 2291
    },
    {
      "epoch": 3.0,
      "learning_rate": 4.027819242278443e-05,
      "logits/chosen": -2.086125135421753,
      "logits/rejected": -2.167119026184082,
      "logps/chosen": -212.73826599121094,
      "logps/rejected": -309.79736328125,
      "loss": 0.1633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.7124667167663574,
      "rewards/margins": 7.573567867279053,
      "rewards/rejected": -11.28603458404541,
      "step": 2292
    },
    {
      "epoch": 3.0,
      "learning_rate": 4.026994616589235e-05,
      "logits/chosen": -2.16123366355896,
      "logits/rejected": -2.154041290283203,
      "logps/chosen": -294.1865234375,
      "logps/rejected": -340.05108642578125,
      "loss": 0.0444,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.380455017089844,
      "rewards/margins": 8.062877655029297,
      "rewards/rejected": -13.44333267211914,
      "step": 2293
    },
    {
      "epoch": 3.0,
      "learning_rate": 4.026169725809289e-05,
      "logits/chosen": -2.189936399459839,
      "logits/rejected": -2.1102802753448486,
      "logps/chosen": -210.06727600097656,
      "logps/rejected": -263.0583190917969,
      "loss": 0.1329,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.176320552825928,
      "rewards/margins": 7.507213592529297,
      "rewards/rejected": -11.68353271484375,
      "step": 2294
    },
    {
      "epoch": 3.0,
      "learning_rate": 4.0253445700818086e-05,
      "logits/chosen": -2.2092671394348145,
      "logits/rejected": -2.2503600120544434,
      "logps/chosen": -234.4791259765625,
      "logps/rejected": -332.8056640625,
      "loss": 0.0437,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.380105018615723,
      "rewards/margins": 9.157236099243164,
      "rewards/rejected": -13.537342071533203,
      "step": 2295
    },
    {
      "epoch": 3.0,
      "learning_rate": 4.0245191495500445e-05,
      "logits/chosen": -2.053370714187622,
      "logits/rejected": -2.0985865592956543,
      "logps/chosen": -239.59951782226562,
      "logps/rejected": -299.9369812011719,
      "loss": 0.0443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.36566162109375,
      "rewards/margins": 7.33683967590332,
      "rewards/rejected": -11.70250129699707,
      "step": 2296
    },
    {
      "epoch": 3.01,
      "learning_rate": 4.023693464357291e-05,
      "logits/chosen": -2.1033010482788086,
      "logits/rejected": -2.139561176300049,
      "logps/chosen": -252.38595581054688,
      "logps/rejected": -339.8417053222656,
      "loss": 0.088,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.987560749053955,
      "rewards/margins": 8.050565719604492,
      "rewards/rejected": -13.038125991821289,
      "step": 2297
    },
    {
      "epoch": 3.01,
      "learning_rate": 4.02286751464689e-05,
      "logits/chosen": -2.361865520477295,
      "logits/rejected": -2.4237165451049805,
      "logps/chosen": -236.89031982421875,
      "logps/rejected": -370.56610107421875,
      "loss": 0.0436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.124394416809082,
      "rewards/margins": 10.090132713317871,
      "rewards/rejected": -14.21452808380127,
      "step": 2298
    },
    {
      "epoch": 3.01,
      "learning_rate": 4.022041300562228e-05,
      "logits/chosen": -1.9574178457260132,
      "logits/rejected": -2.0618011951446533,
      "logps/chosen": -223.8952178955078,
      "logps/rejected": -323.16400146484375,
      "loss": 0.045,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.20233678817749,
      "rewards/margins": 10.032905578613281,
      "rewards/rejected": -14.235241889953613,
      "step": 2299
    },
    {
      "epoch": 3.01,
      "learning_rate": 4.021214822246739e-05,
      "logits/chosen": -2.163879156112671,
      "logits/rejected": -2.2356607913970947,
      "logps/chosen": -258.11541748046875,
      "logps/rejected": -380.8958435058594,
      "loss": 0.046,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.227643966674805,
      "rewards/margins": 8.98115062713623,
      "rewards/rejected": -15.208795547485352,
      "step": 2300
    },
    {
      "epoch": 3.01,
      "learning_rate": 4.020388079843902e-05,
      "logits/chosen": -2.112044334411621,
      "logits/rejected": -2.1053316593170166,
      "logps/chosen": -280.8301696777344,
      "logps/rejected": -317.64324951171875,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.152070045471191,
      "rewards/margins": 8.259082794189453,
      "rewards/rejected": -12.411153793334961,
      "step": 2301
    },
    {
      "epoch": 3.01,
      "learning_rate": 4.019561073497241e-05,
      "logits/chosen": -2.4246768951416016,
      "logits/rejected": -2.440532684326172,
      "logps/chosen": -282.09765625,
      "logps/rejected": -370.8497314453125,
      "loss": 0.0868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.180196762084961,
      "rewards/margins": 8.825722694396973,
      "rewards/rejected": -14.00592041015625,
      "step": 2302
    },
    {
      "epoch": 3.01,
      "learning_rate": 4.0187338033503275e-05,
      "logits/chosen": -2.1134700775146484,
      "logits/rejected": -2.1776392459869385,
      "logps/chosen": -189.08233642578125,
      "logps/rejected": -281.25531005859375,
      "loss": 0.044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.9960646629333496,
      "rewards/margins": 8.551589965820312,
      "rewards/rejected": -11.547654151916504,
      "step": 2303
    },
    {
      "epoch": 3.02,
      "learning_rate": 4.0179062695467784e-05,
      "logits/chosen": -2.1593093872070312,
      "logits/rejected": -2.1546666622161865,
      "logps/chosen": -264.7270812988281,
      "logps/rejected": -367.3416748046875,
      "loss": 0.0453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.1254730224609375,
      "rewards/margins": 7.990652084350586,
      "rewards/rejected": -12.11612606048584,
      "step": 2304
    },
    {
      "epoch": 3.02,
      "learning_rate": 4.017078472230255e-05,
      "logits/chosen": -2.1838388442993164,
      "logits/rejected": -2.2894444465637207,
      "logps/chosen": -263.6777038574219,
      "logps/rejected": -403.84674072265625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.472953796386719,
      "rewards/margins": 11.139884948730469,
      "rewards/rejected": -15.612838745117188,
      "step": 2305
    },
    {
      "epoch": 3.02,
      "learning_rate": 4.016250411544466e-05,
      "logits/chosen": -1.8764675855636597,
      "logits/rejected": -1.9426155090332031,
      "logps/chosen": -236.81805419921875,
      "logps/rejected": -368.4296875,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.013107776641846,
      "rewards/margins": 10.904794692993164,
      "rewards/rejected": -15.917903900146484,
      "step": 2306
    },
    {
      "epoch": 3.02,
      "learning_rate": 4.015422087633165e-05,
      "logits/chosen": -2.1358773708343506,
      "logits/rejected": -2.325611114501953,
      "logps/chosen": -219.749755859375,
      "logps/rejected": -342.8475646972656,
      "loss": 0.0439,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.907919883728027,
      "rewards/margins": 9.476095199584961,
      "rewards/rejected": -14.384015083312988,
      "step": 2307
    },
    {
      "epoch": 3.02,
      "learning_rate": 4.0145935006401516e-05,
      "logits/chosen": -2.153437376022339,
      "logits/rejected": -2.0697827339172363,
      "logps/chosen": -264.7963562011719,
      "logps/rejected": -315.2132873535156,
      "loss": 0.0468,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.331108093261719,
      "rewards/margins": 7.732646465301514,
      "rewards/rejected": -13.06375503540039,
      "step": 2308
    },
    {
      "epoch": 3.02,
      "learning_rate": 4.0137646507092706e-05,
      "logits/chosen": -2.2929344177246094,
      "logits/rejected": -2.3997268676757812,
      "logps/chosen": -272.797119140625,
      "logps/rejected": -399.6045837402344,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.711956977844238,
      "rewards/margins": 11.528371810913086,
      "rewards/rejected": -16.240327835083008,
      "step": 2309
    },
    {
      "epoch": 3.02,
      "learning_rate": 4.012935537984414e-05,
      "logits/chosen": -2.355783224105835,
      "logits/rejected": -2.362971544265747,
      "logps/chosen": -311.9186706542969,
      "logps/rejected": -427.03094482421875,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.685152053833008,
      "rewards/margins": 10.624361038208008,
      "rewards/rejected": -16.309513092041016,
      "step": 2310
    },
    {
      "epoch": 3.02,
      "learning_rate": 4.0121061626095155e-05,
      "logits/chosen": -2.1310548782348633,
      "logits/rejected": -2.0894548892974854,
      "logps/chosen": -234.70578002929688,
      "logps/rejected": -341.914794921875,
      "loss": 0.0438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.285902976989746,
      "rewards/margins": 8.906684875488281,
      "rewards/rejected": -14.192588806152344,
      "step": 2311
    },
    {
      "epoch": 3.03,
      "learning_rate": 4.0112765247285595e-05,
      "logits/chosen": -2.230635643005371,
      "logits/rejected": -2.2372360229492188,
      "logps/chosen": -205.48318481445312,
      "logps/rejected": -288.3927917480469,
      "loss": 0.044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.535229206085205,
      "rewards/margins": 8.466339111328125,
      "rewards/rejected": -13.001565933227539,
      "step": 2312
    },
    {
      "epoch": 3.03,
      "learning_rate": 4.010446624485572e-05,
      "logits/chosen": -2.2720067501068115,
      "logits/rejected": -2.2585554122924805,
      "logps/chosen": -296.4164123535156,
      "logps/rejected": -356.4053039550781,
      "loss": 0.0438,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.370475769042969,
      "rewards/margins": 8.456291198730469,
      "rewards/rejected": -13.826766014099121,
      "step": 2313
    },
    {
      "epoch": 3.03,
      "learning_rate": 4.009616462024628e-05,
      "logits/chosen": -2.039180278778076,
      "logits/rejected": -2.0593509674072266,
      "logps/chosen": -213.3941650390625,
      "logps/rejected": -323.753173828125,
      "loss": 0.0873,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.562514781951904,
      "rewards/margins": 8.90079402923584,
      "rewards/rejected": -13.463308334350586,
      "step": 2314
    },
    {
      "epoch": 3.03,
      "learning_rate": 4.0087860374898446e-05,
      "logits/chosen": -2.219299793243408,
      "logits/rejected": -2.2711002826690674,
      "logps/chosen": -201.03753662109375,
      "logps/rejected": -316.4630126953125,
      "loss": 0.087,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.167980670928955,
      "rewards/margins": 8.080424308776855,
      "rewards/rejected": -12.248404502868652,
      "step": 2315
    },
    {
      "epoch": 3.03,
      "learning_rate": 4.007955351025387e-05,
      "logits/chosen": -2.040846824645996,
      "logits/rejected": -2.157193183898926,
      "logps/chosen": -245.75889587402344,
      "logps/rejected": -385.9333801269531,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.451557159423828,
      "rewards/margins": 11.200770378112793,
      "rewards/rejected": -16.652326583862305,
      "step": 2316
    },
    {
      "epoch": 3.03,
      "learning_rate": 4.007124402775463e-05,
      "logits/chosen": -2.101086378097534,
      "logits/rejected": -2.293389320373535,
      "logps/chosen": -208.45314025878906,
      "logps/rejected": -346.80401611328125,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.186187744140625,
      "rewards/margins": 10.941791534423828,
      "rewards/rejected": -16.127979278564453,
      "step": 2317
    },
    {
      "epoch": 3.03,
      "learning_rate": 4.00629319288433e-05,
      "logits/chosen": -1.848655343055725,
      "logits/rejected": -1.9465444087982178,
      "logps/chosen": -278.0640563964844,
      "logps/rejected": -320.0218200683594,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.153502941131592,
      "rewards/margins": 7.768182754516602,
      "rewards/rejected": -12.921686172485352,
      "step": 2318
    },
    {
      "epoch": 3.03,
      "learning_rate": 4.005461721496286e-05,
      "logits/chosen": -1.8471152782440186,
      "logits/rejected": -1.9216175079345703,
      "logps/chosen": -219.87152099609375,
      "logps/rejected": -335.74462890625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6926662921905518,
      "rewards/margins": 10.610774993896484,
      "rewards/rejected": -14.30344295501709,
      "step": 2319
    },
    {
      "epoch": 3.04,
      "learning_rate": 4.00462998875568e-05,
      "logits/chosen": -2.17388916015625,
      "logits/rejected": -2.2358853816986084,
      "logps/chosen": -244.40786743164062,
      "logps/rejected": -344.83514404296875,
      "loss": 0.0064,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.317914009094238,
      "rewards/margins": 9.465472221374512,
      "rewards/rejected": -14.78338623046875,
      "step": 2320
    },
    {
      "epoch": 3.04,
      "learning_rate": 4.0037979948069003e-05,
      "logits/chosen": -2.2021055221557617,
      "logits/rejected": -2.250926971435547,
      "logps/chosen": -189.5944366455078,
      "logps/rejected": -303.94281005859375,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.41383695602417,
      "rewards/margins": 9.51795768737793,
      "rewards/rejected": -13.931794166564941,
      "step": 2321
    },
    {
      "epoch": 3.04,
      "learning_rate": 4.002965739794385e-05,
      "logits/chosen": -1.741159439086914,
      "logits/rejected": -1.768996238708496,
      "logps/chosen": -258.5458984375,
      "logps/rejected": -308.5641784667969,
      "loss": 0.044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.688573360443115,
      "rewards/margins": 8.116438865661621,
      "rewards/rejected": -12.805011749267578,
      "step": 2322
    },
    {
      "epoch": 3.04,
      "learning_rate": 4.002133223862615e-05,
      "logits/chosen": -2.1633663177490234,
      "logits/rejected": -2.1048080921173096,
      "logps/chosen": -210.6006317138672,
      "logps/rejected": -328.53546142578125,
      "loss": 0.045,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.308881759643555,
      "rewards/margins": 9.225309371948242,
      "rewards/rejected": -13.534191131591797,
      "step": 2323
    },
    {
      "epoch": 3.04,
      "learning_rate": 4.001300447156119e-05,
      "logits/chosen": -1.9731532335281372,
      "logits/rejected": -2.082718849182129,
      "logps/chosen": -184.78358459472656,
      "logps/rejected": -326.98468017578125,
      "loss": 0.0437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.704720497131348,
      "rewards/margins": 9.48631763458252,
      "rewards/rejected": -14.191038131713867,
      "step": 2324
    },
    {
      "epoch": 3.04,
      "learning_rate": 4.0004674098194675e-05,
      "logits/chosen": -2.027649402618408,
      "logits/rejected": -2.1711618900299072,
      "logps/chosen": -218.40965270996094,
      "logps/rejected": -353.84210205078125,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.636758804321289,
      "rewards/margins": 8.774707794189453,
      "rewards/rejected": -14.411465644836426,
      "step": 2325
    },
    {
      "epoch": 3.04,
      "learning_rate": 3.9996341119972805e-05,
      "logits/chosen": -2.0298938751220703,
      "logits/rejected": -2.186424732208252,
      "logps/chosen": -188.10995483398438,
      "logps/rejected": -320.994140625,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.797072410583496,
      "rewards/margins": 9.481252670288086,
      "rewards/rejected": -14.278324127197266,
      "step": 2326
    },
    {
      "epoch": 3.05,
      "learning_rate": 3.998800553834219e-05,
      "logits/chosen": -1.8152039051055908,
      "logits/rejected": -1.8558685779571533,
      "logps/chosen": -211.7963409423828,
      "logps/rejected": -309.7165832519531,
      "loss": 0.0569,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.820761203765869,
      "rewards/margins": 10.48230266571045,
      "rewards/rejected": -15.30306339263916,
      "step": 2327
    },
    {
      "epoch": 3.05,
      "learning_rate": 3.9979667354749917e-05,
      "logits/chosen": -2.2250239849090576,
      "logits/rejected": -2.2089879512786865,
      "logps/chosen": -243.59071350097656,
      "logps/rejected": -316.2306213378906,
      "loss": 0.0489,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.653672218322754,
      "rewards/margins": 9.42144775390625,
      "rewards/rejected": -14.075119018554688,
      "step": 2328
    },
    {
      "epoch": 3.05,
      "learning_rate": 3.997132657064353e-05,
      "logits/chosen": -1.8166918754577637,
      "logits/rejected": -1.8703713417053223,
      "logps/chosen": -256.21514892578125,
      "logps/rejected": -344.1813049316406,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.947235107421875,
      "rewards/margins": 9.703486442565918,
      "rewards/rejected": -14.65072250366211,
      "step": 2329
    },
    {
      "epoch": 3.05,
      "learning_rate": 3.9962983187470994e-05,
      "logits/chosen": -2.1914587020874023,
      "logits/rejected": -2.188868761062622,
      "logps/chosen": -211.61862182617188,
      "logps/rejected": -278.4850158691406,
      "loss": 0.09,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.627830505371094,
      "rewards/margins": 7.591188430786133,
      "rewards/rejected": -12.219019889831543,
      "step": 2330
    },
    {
      "epoch": 3.05,
      "learning_rate": 3.995463720668076e-05,
      "logits/chosen": -2.0139284133911133,
      "logits/rejected": -1.872101068496704,
      "logps/chosen": -264.0347595214844,
      "logps/rejected": -338.0701904296875,
      "loss": 0.0435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.595579624176025,
      "rewards/margins": 9.703751564025879,
      "rewards/rejected": -15.299330711364746,
      "step": 2331
    },
    {
      "epoch": 3.05,
      "learning_rate": 3.994628862972172e-05,
      "logits/chosen": -2.0300374031066895,
      "logits/rejected": -2.114473342895508,
      "logps/chosen": -224.14810180664062,
      "logps/rejected": -363.4672546386719,
      "loss": 0.0867,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.546936988830566,
      "rewards/margins": 9.266657829284668,
      "rewards/rejected": -14.813594818115234,
      "step": 2332
    },
    {
      "epoch": 3.05,
      "learning_rate": 3.993793745804319e-05,
      "logits/chosen": -2.044935941696167,
      "logits/rejected": -2.18890380859375,
      "logps/chosen": -250.87396240234375,
      "logps/rejected": -357.27008056640625,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.594356060028076,
      "rewards/margins": 9.66875171661377,
      "rewards/rejected": -15.263107299804688,
      "step": 2333
    },
    {
      "epoch": 3.05,
      "learning_rate": 3.9929583693094967e-05,
      "logits/chosen": -2.0749664306640625,
      "logits/rejected": -2.10062575340271,
      "logps/chosen": -223.5476837158203,
      "logps/rejected": -336.767578125,
      "loss": 0.0435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.567058563232422,
      "rewards/margins": 9.905509948730469,
      "rewards/rejected": -14.47256851196289,
      "step": 2334
    },
    {
      "epoch": 3.06,
      "learning_rate": 3.9921227336327293e-05,
      "logits/chosen": -2.249016284942627,
      "logits/rejected": -2.2628750801086426,
      "logps/chosen": -282.121826171875,
      "logps/rejected": -331.7041015625,
      "loss": 0.0437,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.705267429351807,
      "rewards/margins": 9.401190757751465,
      "rewards/rejected": -14.10645866394043,
      "step": 2335
    },
    {
      "epoch": 3.06,
      "learning_rate": 3.991286838919086e-05,
      "logits/chosen": -2.0916032791137695,
      "logits/rejected": -2.139112949371338,
      "logps/chosen": -226.25941467285156,
      "logps/rejected": -367.0303649902344,
      "loss": 0.0439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.4411211013793945,
      "rewards/margins": 11.064277648925781,
      "rewards/rejected": -15.505398750305176,
      "step": 2336
    },
    {
      "epoch": 3.06,
      "learning_rate": 3.9904506853136796e-05,
      "logits/chosen": -2.2027699947357178,
      "logits/rejected": -2.300292730331421,
      "logps/chosen": -265.5385437011719,
      "logps/rejected": -397.46527099609375,
      "loss": 0.0449,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.148678779602051,
      "rewards/margins": 9.942577362060547,
      "rewards/rejected": -16.091257095336914,
      "step": 2337
    },
    {
      "epoch": 3.06,
      "learning_rate": 3.989614272961669e-05,
      "logits/chosen": -2.186069965362549,
      "logits/rejected": -2.189955234527588,
      "logps/chosen": -238.25157165527344,
      "logps/rejected": -309.65679931640625,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.0187602043151855,
      "rewards/margins": 7.825289249420166,
      "rewards/rejected": -13.844049453735352,
      "step": 2338
    },
    {
      "epoch": 3.06,
      "learning_rate": 3.988777602008259e-05,
      "logits/chosen": -2.190187931060791,
      "logits/rejected": -2.195145845413208,
      "logps/chosen": -252.57415771484375,
      "logps/rejected": -355.80804443359375,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.373104572296143,
      "rewards/margins": 11.054689407348633,
      "rewards/rejected": -15.427794456481934,
      "step": 2339
    },
    {
      "epoch": 3.06,
      "learning_rate": 3.9879406725986965e-05,
      "logits/chosen": -1.855334997177124,
      "logits/rejected": -1.9873031377792358,
      "logps/chosen": -205.4321746826172,
      "logps/rejected": -339.5233154296875,
      "loss": 0.0868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.916016578674316,
      "rewards/margins": 8.870521545410156,
      "rewards/rejected": -13.786538124084473,
      "step": 2340
    },
    {
      "epoch": 3.06,
      "learning_rate": 3.987103484878275e-05,
      "logits/chosen": -2.270313262939453,
      "logits/rejected": -2.252146005630493,
      "logps/chosen": -253.8924102783203,
      "logps/rejected": -356.2812194824219,
      "loss": 0.0435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.975635528564453,
      "rewards/margins": 9.70572566986084,
      "rewards/rejected": -15.681361198425293,
      "step": 2341
    },
    {
      "epoch": 3.06,
      "learning_rate": 3.986266038992335e-05,
      "logits/chosen": -2.0620858669281006,
      "logits/rejected": -2.1372265815734863,
      "logps/chosen": -224.60153198242188,
      "logps/rejected": -314.62744140625,
      "loss": 0.044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.936737060546875,
      "rewards/margins": 9.658467292785645,
      "rewards/rejected": -14.595205307006836,
      "step": 2342
    },
    {
      "epoch": 3.07,
      "learning_rate": 3.9854283350862564e-05,
      "logits/chosen": -2.163487434387207,
      "logits/rejected": -2.112785816192627,
      "logps/chosen": -256.3828125,
      "logps/rejected": -359.2196044921875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.521401405334473,
      "rewards/margins": 9.490942001342773,
      "rewards/rejected": -15.012344360351562,
      "step": 2343
    },
    {
      "epoch": 3.07,
      "learning_rate": 3.984590373305469e-05,
      "logits/chosen": -2.2192397117614746,
      "logits/rejected": -2.2327206134796143,
      "logps/chosen": -219.90838623046875,
      "logps/rejected": -313.4751892089844,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.904820442199707,
      "rewards/margins": 9.546765327453613,
      "rewards/rejected": -14.45158576965332,
      "step": 2344
    },
    {
      "epoch": 3.07,
      "learning_rate": 3.983752153795445e-05,
      "logits/chosen": -2.0529139041900635,
      "logits/rejected": -2.1299502849578857,
      "logps/chosen": -209.40573120117188,
      "logps/rejected": -321.980224609375,
      "loss": 0.0872,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.771730899810791,
      "rewards/margins": 8.564140319824219,
      "rewards/rejected": -14.335870742797852,
      "step": 2345
    },
    {
      "epoch": 3.07,
      "learning_rate": 3.982913676701702e-05,
      "logits/chosen": -2.1710922718048096,
      "logits/rejected": -2.21834135055542,
      "logps/chosen": -253.80889892578125,
      "logps/rejected": -362.06597900390625,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.591020107269287,
      "rewards/margins": 10.34268569946289,
      "rewards/rejected": -15.93370532989502,
      "step": 2346
    },
    {
      "epoch": 3.07,
      "learning_rate": 3.982074942169801e-05,
      "logits/chosen": -2.0652060508728027,
      "logits/rejected": -2.0282676219940186,
      "logps/chosen": -297.8626708984375,
      "logps/rejected": -414.75244140625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.402783393859863,
      "rewards/margins": 10.836182594299316,
      "rewards/rejected": -16.23896598815918,
      "step": 2347
    },
    {
      "epoch": 3.07,
      "learning_rate": 3.98123595034535e-05,
      "logits/chosen": -1.8497111797332764,
      "logits/rejected": -1.9697858095169067,
      "logps/chosen": -196.43084716796875,
      "logps/rejected": -325.74005126953125,
      "loss": 0.0867,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.046850204467773,
      "rewards/margins": 9.167963027954102,
      "rewards/rejected": -14.214814186096191,
      "step": 2348
    },
    {
      "epoch": 3.07,
      "learning_rate": 3.980396701374001e-05,
      "logits/chosen": -2.0902960300445557,
      "logits/rejected": -2.1278557777404785,
      "logps/chosen": -211.67010498046875,
      "logps/rejected": -327.716552734375,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.313720226287842,
      "rewards/margins": 9.680084228515625,
      "rewards/rejected": -14.993803977966309,
      "step": 2349
    },
    {
      "epoch": 3.08,
      "learning_rate": 3.979557195401447e-05,
      "logits/chosen": -2.1743905544281006,
      "logits/rejected": -2.2324106693267822,
      "logps/chosen": -234.66226196289062,
      "logps/rejected": -340.6358947753906,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.393934726715088,
      "rewards/margins": 8.925092697143555,
      "rewards/rejected": -14.319025039672852,
      "step": 2350
    },
    {
      "epoch": 3.08,
      "learning_rate": 3.978717432573432e-05,
      "logits/chosen": -1.8415966033935547,
      "logits/rejected": -1.935300350189209,
      "logps/chosen": -213.11997985839844,
      "logps/rejected": -358.7032470703125,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.9846391677856445,
      "rewards/margins": 10.274259567260742,
      "rewards/rejected": -16.258899688720703,
      "step": 2351
    },
    {
      "epoch": 3.08,
      "learning_rate": 3.9778774130357396e-05,
      "logits/chosen": -2.091505527496338,
      "logits/rejected": -2.1661930084228516,
      "logps/chosen": -209.1724853515625,
      "logps/rejected": -393.89544677734375,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.482723236083984,
      "rewards/margins": 10.429054260253906,
      "rewards/rejected": -16.911779403686523,
      "step": 2352
    },
    {
      "epoch": 3.08,
      "learning_rate": 3.977037136934201e-05,
      "logits/chosen": -2.0253586769104004,
      "logits/rejected": -2.018190622329712,
      "logps/chosen": -230.11798095703125,
      "logps/rejected": -301.07843017578125,
      "loss": 0.0868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.083752155303955,
      "rewards/margins": 9.406051635742188,
      "rewards/rejected": -14.489802360534668,
      "step": 2353
    },
    {
      "epoch": 3.08,
      "learning_rate": 3.976196604414689e-05,
      "logits/chosen": -1.952399730682373,
      "logits/rejected": -1.9626762866973877,
      "logps/chosen": -261.0067138671875,
      "logps/rejected": -397.1845397949219,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.120563983917236,
      "rewards/margins": 10.943238258361816,
      "rewards/rejected": -16.06380271911621,
      "step": 2354
    },
    {
      "epoch": 3.08,
      "learning_rate": 3.975355815623124e-05,
      "logits/chosen": -1.919739007949829,
      "logits/rejected": -1.9575756788253784,
      "logps/chosen": -195.85140991210938,
      "logps/rejected": -286.96673583984375,
      "loss": 0.0439,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.722510814666748,
      "rewards/margins": 8.700567245483398,
      "rewards/rejected": -13.423078536987305,
      "step": 2355
    },
    {
      "epoch": 3.08,
      "learning_rate": 3.974514770705468e-05,
      "logits/chosen": -2.0669007301330566,
      "logits/rejected": -1.9612680673599243,
      "logps/chosen": -219.40257263183594,
      "logps/rejected": -338.7507629394531,
      "loss": 0.0479,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.551650524139404,
      "rewards/margins": 9.081823348999023,
      "rewards/rejected": -14.633474349975586,
      "step": 2356
    },
    {
      "epoch": 3.08,
      "learning_rate": 3.973673469807729e-05,
      "logits/chosen": -2.1750898361206055,
      "logits/rejected": -2.220430612564087,
      "logps/chosen": -227.869384765625,
      "logps/rejected": -315.6466064453125,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.440352916717529,
      "rewards/margins": 9.903848648071289,
      "rewards/rejected": -15.344200134277344,
      "step": 2357
    },
    {
      "epoch": 3.09,
      "learning_rate": 3.972831913075961e-05,
      "logits/chosen": -2.1814780235290527,
      "logits/rejected": -2.2241060733795166,
      "logps/chosen": -233.10333251953125,
      "logps/rejected": -341.2297668457031,
      "loss": 0.0528,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.034167289733887,
      "rewards/margins": 9.180243492126465,
      "rewards/rejected": -15.214411735534668,
      "step": 2358
    },
    {
      "epoch": 3.09,
      "learning_rate": 3.971990100656258e-05,
      "logits/chosen": -2.333792209625244,
      "logits/rejected": -2.349745035171509,
      "logps/chosen": -231.09255981445312,
      "logps/rejected": -278.3640441894531,
      "loss": 0.132,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.1729912757873535,
      "rewards/margins": 7.631937026977539,
      "rewards/rejected": -12.80492877960205,
      "step": 2359
    },
    {
      "epoch": 3.09,
      "learning_rate": 3.9711480326947636e-05,
      "logits/chosen": -1.8499326705932617,
      "logits/rejected": -1.9601032733917236,
      "logps/chosen": -144.95199584960938,
      "logps/rejected": -281.6170654296875,
      "loss": 0.1316,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.8826472759246826,
      "rewards/margins": 8.919089317321777,
      "rewards/rejected": -12.801735877990723,
      "step": 2360
    },
    {
      "epoch": 3.09,
      "learning_rate": 3.970305709337662e-05,
      "logits/chosen": -1.744267463684082,
      "logits/rejected": -1.7960014343261719,
      "logps/chosen": -278.4100646972656,
      "logps/rejected": -379.42315673828125,
      "loss": 0.05,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.190540790557861,
      "rewards/margins": 8.991704940795898,
      "rewards/rejected": -15.182245254516602,
      "step": 2361
    },
    {
      "epoch": 3.09,
      "learning_rate": 3.969463130731183e-05,
      "logits/chosen": -1.9131689071655273,
      "logits/rejected": -1.9178646802902222,
      "logps/chosen": -238.28317260742188,
      "logps/rejected": -336.04132080078125,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.83204460144043,
      "rewards/margins": 9.937345504760742,
      "rewards/rejected": -15.769390106201172,
      "step": 2362
    },
    {
      "epoch": 3.09,
      "learning_rate": 3.968620297021601e-05,
      "logits/chosen": -2.2412827014923096,
      "logits/rejected": -2.295560598373413,
      "logps/chosen": -256.8214111328125,
      "logps/rejected": -343.5921936035156,
      "loss": 0.0568,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.132884502410889,
      "rewards/margins": 8.71896743774414,
      "rewards/rejected": -15.851852416992188,
      "step": 2363
    },
    {
      "epoch": 3.09,
      "learning_rate": 3.9677772083552345e-05,
      "logits/chosen": -1.8722002506256104,
      "logits/rejected": -2.006021499633789,
      "logps/chosen": -205.1575469970703,
      "logps/rejected": -341.50830078125,
      "loss": 0.0436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.86874532699585,
      "rewards/margins": 10.063451766967773,
      "rewards/rejected": -14.932196617126465,
      "step": 2364
    },
    {
      "epoch": 3.1,
      "learning_rate": 3.966933864878446e-05,
      "logits/chosen": -2.046663522720337,
      "logits/rejected": -2.1174910068511963,
      "logps/chosen": -206.85365295410156,
      "logps/rejected": -294.822998046875,
      "loss": 0.1311,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -6.025838375091553,
      "rewards/margins": 7.51859188079834,
      "rewards/rejected": -13.54443073272705,
      "step": 2365
    },
    {
      "epoch": 3.1,
      "learning_rate": 3.966090266737643e-05,
      "logits/chosen": -2.019179582595825,
      "logits/rejected": -1.9590449333190918,
      "logps/chosen": -188.83555603027344,
      "logps/rejected": -286.6895751953125,
      "loss": 0.0448,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.439995288848877,
      "rewards/margins": 9.44103717803955,
      "rewards/rejected": -13.881032943725586,
      "step": 2366
    },
    {
      "epoch": 3.1,
      "learning_rate": 3.9652464140792754e-05,
      "logits/chosen": -1.8752981424331665,
      "logits/rejected": -2.010054588317871,
      "logps/chosen": -241.266845703125,
      "logps/rejected": -383.7904357910156,
      "loss": 0.0436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.881939888000488,
      "rewards/margins": 10.096649169921875,
      "rewards/rejected": -16.978588104248047,
      "step": 2367
    },
    {
      "epoch": 3.1,
      "learning_rate": 3.9644023070498394e-05,
      "logits/chosen": -1.8323734998703003,
      "logits/rejected": -1.8427077531814575,
      "logps/chosen": -206.6799774169922,
      "logps/rejected": -313.88092041015625,
      "loss": 0.0877,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.858495235443115,
      "rewards/margins": 8.448741912841797,
      "rewards/rejected": -13.30723762512207,
      "step": 2368
    },
    {
      "epoch": 3.1,
      "learning_rate": 3.963557945795874e-05,
      "logits/chosen": -2.239041328430176,
      "logits/rejected": -2.2368831634521484,
      "logps/chosen": -275.11798095703125,
      "logps/rejected": -377.6287841796875,
      "loss": 0.0454,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.356086730957031,
      "rewards/margins": 10.835125923156738,
      "rewards/rejected": -17.191213607788086,
      "step": 2369
    },
    {
      "epoch": 3.1,
      "learning_rate": 3.962713330463963e-05,
      "logits/chosen": -2.091503620147705,
      "logits/rejected": -2.0818939208984375,
      "logps/chosen": -335.230224609375,
      "logps/rejected": -415.01788330078125,
      "loss": 0.0868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.815808296203613,
      "rewards/margins": 8.49116325378418,
      "rewards/rejected": -14.306971549987793,
      "step": 2370
    },
    {
      "epoch": 3.1,
      "learning_rate": 3.961868461200734e-05,
      "logits/chosen": -2.1107940673828125,
      "logits/rejected": -2.1707143783569336,
      "logps/chosen": -236.53013610839844,
      "logps/rejected": -324.61712646484375,
      "loss": 0.0524,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.84749698638916,
      "rewards/margins": 9.747594833374023,
      "rewards/rejected": -15.595091819763184,
      "step": 2371
    },
    {
      "epoch": 3.1,
      "learning_rate": 3.961023338152859e-05,
      "logits/chosen": -2.094388008117676,
      "logits/rejected": -2.1055898666381836,
      "logps/chosen": -271.1788635253906,
      "logps/rejected": -337.4134216308594,
      "loss": 0.044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.544390678405762,
      "rewards/margins": 8.601953506469727,
      "rewards/rejected": -14.146343231201172,
      "step": 2372
    },
    {
      "epoch": 3.11,
      "learning_rate": 3.960177961467053e-05,
      "logits/chosen": -1.7063359022140503,
      "logits/rejected": -1.7682796716690063,
      "logps/chosen": -256.10052490234375,
      "logps/rejected": -396.14739990234375,
      "loss": 0.0436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.337587356567383,
      "rewards/margins": 10.831178665161133,
      "rewards/rejected": -17.168766021728516,
      "step": 2373
    },
    {
      "epoch": 3.11,
      "learning_rate": 3.9593323312900776e-05,
      "logits/chosen": -1.904474139213562,
      "logits/rejected": -1.9390546083450317,
      "logps/chosen": -229.9960174560547,
      "logps/rejected": -338.8091735839844,
      "loss": 0.0868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.166641712188721,
      "rewards/margins": 8.765958786010742,
      "rewards/rejected": -14.932600975036621,
      "step": 2374
    },
    {
      "epoch": 3.11,
      "learning_rate": 3.958486447768736e-05,
      "logits/chosen": -2.048222064971924,
      "logits/rejected": -2.0803263187408447,
      "logps/chosen": -253.60183715820312,
      "logps/rejected": -351.0816345214844,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.731586933135986,
      "rewards/margins": 10.426437377929688,
      "rewards/rejected": -16.15802574157715,
      "step": 2375
    },
    {
      "epoch": 3.11,
      "learning_rate": 3.957640311049874e-05,
      "logits/chosen": -2.3471922874450684,
      "logits/rejected": -2.3304343223571777,
      "logps/chosen": -250.97804260253906,
      "logps/rejected": -310.53143310546875,
      "loss": 0.0462,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.121223449707031,
      "rewards/margins": 8.97576904296875,
      "rewards/rejected": -14.096992492675781,
      "step": 2376
    },
    {
      "epoch": 3.11,
      "learning_rate": 3.956793921280386e-05,
      "logits/chosen": -2.0674235820770264,
      "logits/rejected": -2.17264461517334,
      "logps/chosen": -251.26593017578125,
      "logps/rejected": -397.5065612792969,
      "loss": 0.0434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.873909950256348,
      "rewards/margins": 11.099771499633789,
      "rewards/rejected": -16.973682403564453,
      "step": 2377
    },
    {
      "epoch": 3.11,
      "learning_rate": 3.9559472786072064e-05,
      "logits/chosen": -1.8911620378494263,
      "logits/rejected": -2.07753324508667,
      "logps/chosen": -220.3954620361328,
      "logps/rejected": -392.901611328125,
      "loss": 0.0436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.4543890953063965,
      "rewards/margins": 11.461380004882812,
      "rewards/rejected": -16.915769577026367,
      "step": 2378
    },
    {
      "epoch": 3.11,
      "learning_rate": 3.955100383177316e-05,
      "logits/chosen": -2.0628786087036133,
      "logits/rejected": -2.064690351486206,
      "logps/chosen": -277.8653564453125,
      "logps/rejected": -342.07745361328125,
      "loss": 0.0885,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.13007926940918,
      "rewards/margins": 7.721760272979736,
      "rewards/rejected": -12.851840019226074,
      "step": 2379
    },
    {
      "epoch": 3.11,
      "learning_rate": 3.954253235137737e-05,
      "logits/chosen": -2.1326215267181396,
      "logits/rejected": -2.1177854537963867,
      "logps/chosen": -293.449462890625,
      "logps/rejected": -377.8431091308594,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.3262529373168945,
      "rewards/margins": 10.278807640075684,
      "rewards/rejected": -15.605061531066895,
      "step": 2380
    },
    {
      "epoch": 3.12,
      "learning_rate": 3.9534058346355374e-05,
      "logits/chosen": -2.0535805225372314,
      "logits/rejected": -2.0691559314727783,
      "logps/chosen": -250.04638671875,
      "logps/rejected": -352.6424560546875,
      "loss": 0.0873,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.745578765869141,
      "rewards/margins": 8.711980819702148,
      "rewards/rejected": -14.457559585571289,
      "step": 2381
    },
    {
      "epoch": 3.12,
      "learning_rate": 3.952558181817829e-05,
      "logits/chosen": -2.056554079055786,
      "logits/rejected": -2.154934883117676,
      "logps/chosen": -237.72950744628906,
      "logps/rejected": -355.5850830078125,
      "loss": 0.0875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.244574546813965,
      "rewards/margins": 8.941149711608887,
      "rewards/rejected": -15.185724258422852,
      "step": 2382
    },
    {
      "epoch": 3.12,
      "learning_rate": 3.9517102768317654e-05,
      "logits/chosen": -1.9855201244354248,
      "logits/rejected": -2.192671060562134,
      "logps/chosen": -253.00885009765625,
      "logps/rejected": -392.8494873046875,
      "loss": 0.0435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.224951267242432,
      "rewards/margins": 10.622590065002441,
      "rewards/rejected": -16.847543716430664,
      "step": 2383
    },
    {
      "epoch": 3.12,
      "learning_rate": 3.9508621198245467e-05,
      "logits/chosen": -2.1092135906219482,
      "logits/rejected": -2.1657583713531494,
      "logps/chosen": -221.89297485351562,
      "logps/rejected": -368.66650390625,
      "loss": 0.0435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.002373695373535,
      "rewards/margins": 10.422560691833496,
      "rewards/rejected": -15.424935340881348,
      "step": 2384
    },
    {
      "epoch": 3.12,
      "learning_rate": 3.950013710943415e-05,
      "logits/chosen": -2.1366307735443115,
      "logits/rejected": -2.127321720123291,
      "logps/chosen": -258.19921875,
      "logps/rejected": -347.7055358886719,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.898001670837402,
      "rewards/margins": 10.462735176086426,
      "rewards/rejected": -15.360738754272461,
      "step": 2385
    },
    {
      "epoch": 3.12,
      "learning_rate": 3.9491650503356556e-05,
      "logits/chosen": -2.008326530456543,
      "logits/rejected": -1.9478611946105957,
      "logps/chosen": -232.56173706054688,
      "logps/rejected": -329.6402893066406,
      "loss": 0.0435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.509620666503906,
      "rewards/margins": 9.250005722045898,
      "rewards/rejected": -14.759627342224121,
      "step": 2386
    },
    {
      "epoch": 3.12,
      "learning_rate": 3.9483161381486e-05,
      "logits/chosen": -1.8713055849075317,
      "logits/rejected": -1.9137439727783203,
      "logps/chosen": -239.3643798828125,
      "logps/rejected": -343.9569396972656,
      "loss": 0.1734,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -6.358916759490967,
      "rewards/margins": 7.795135974884033,
      "rewards/rejected": -14.154053688049316,
      "step": 2387
    },
    {
      "epoch": 3.13,
      "learning_rate": 3.9474669745296225e-05,
      "logits/chosen": -2.0420758724212646,
      "logits/rejected": -2.1169066429138184,
      "logps/chosen": -265.346923828125,
      "logps/rejected": -364.44671630859375,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.088667392730713,
      "rewards/margins": 9.89621353149414,
      "rewards/rejected": -15.984879493713379,
      "step": 2388
    },
    {
      "epoch": 3.13,
      "learning_rate": 3.946617559626137e-05,
      "logits/chosen": -1.9690918922424316,
      "logits/rejected": -1.896228313446045,
      "logps/chosen": -244.475341796875,
      "logps/rejected": -319.3477783203125,
      "loss": 0.0868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.9726781845092773,
      "rewards/margins": 9.650399208068848,
      "rewards/rejected": -13.623077392578125,
      "step": 2389
    },
    {
      "epoch": 3.13,
      "learning_rate": 3.945767893585608e-05,
      "logits/chosen": -1.9241656064987183,
      "logits/rejected": -2.1013247966766357,
      "logps/chosen": -210.3724365234375,
      "logps/rejected": -335.97918701171875,
      "loss": 0.0871,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.849796295166016,
      "rewards/margins": 9.42531967163086,
      "rewards/rejected": -15.275117874145508,
      "step": 2390
    },
    {
      "epoch": 3.13,
      "learning_rate": 3.9449179765555375e-05,
      "logits/chosen": -2.0780909061431885,
      "logits/rejected": -2.051337718963623,
      "logps/chosen": -258.7431335449219,
      "logps/rejected": -354.255859375,
      "loss": 0.0871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.664587497711182,
      "rewards/margins": 9.591286659240723,
      "rewards/rejected": -15.25587272644043,
      "step": 2391
    },
    {
      "epoch": 3.13,
      "learning_rate": 3.944067808683475e-05,
      "logits/chosen": -2.2163031101226807,
      "logits/rejected": -2.2428455352783203,
      "logps/chosen": -234.6023712158203,
      "logps/rejected": -338.6448059082031,
      "loss": 0.0435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.279127597808838,
      "rewards/margins": 9.972273826599121,
      "rewards/rejected": -16.251399993896484,
      "step": 2392
    },
    {
      "epoch": 3.13,
      "learning_rate": 3.943217390117012e-05,
      "logits/chosen": -2.0962891578674316,
      "logits/rejected": -2.1283798217773438,
      "logps/chosen": -218.8201141357422,
      "logps/rejected": -331.392578125,
      "loss": 0.0437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.476904392242432,
      "rewards/margins": 9.991121292114258,
      "rewards/rejected": -15.468027114868164,
      "step": 2393
    },
    {
      "epoch": 3.13,
      "learning_rate": 3.942366721003783e-05,
      "logits/chosen": -2.068873405456543,
      "logits/rejected": -2.2088582515716553,
      "logps/chosen": -222.7272186279297,
      "logps/rejected": -379.4482421875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.834370136260986,
      "rewards/margins": 11.389117240905762,
      "rewards/rejected": -16.223487854003906,
      "step": 2394
    },
    {
      "epoch": 3.13,
      "learning_rate": 3.941515801491468e-05,
      "logits/chosen": -2.146162509918213,
      "logits/rejected": -2.0940680503845215,
      "logps/chosen": -268.0540771484375,
      "logps/rejected": -352.9101867675781,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.3566179275512695,
      "rewards/margins": 10.106650352478027,
      "rewards/rejected": -15.463268280029297,
      "step": 2395
    },
    {
      "epoch": 3.14,
      "learning_rate": 3.940664631727787e-05,
      "logits/chosen": -2.075049638748169,
      "logits/rejected": -2.1905934810638428,
      "logps/chosen": -263.863037109375,
      "logps/rejected": -367.4610595703125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.505448341369629,
      "rewards/margins": 10.4701566696167,
      "rewards/rejected": -15.975607872009277,
      "step": 2396
    },
    {
      "epoch": 3.14,
      "learning_rate": 3.939813211860507e-05,
      "logits/chosen": -2.1090781688690186,
      "logits/rejected": -2.1186399459838867,
      "logps/chosen": -218.42019653320312,
      "logps/rejected": -285.690185546875,
      "loss": 0.0887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.306107997894287,
      "rewards/margins": 7.967566013336182,
      "rewards/rejected": -13.273674964904785,
      "step": 2397
    },
    {
      "epoch": 3.14,
      "learning_rate": 3.938961542037437e-05,
      "logits/chosen": -1.9859368801116943,
      "logits/rejected": -1.9667267799377441,
      "logps/chosen": -231.34140014648438,
      "logps/rejected": -309.3080749511719,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.32011604309082,
      "rewards/margins": 10.557493209838867,
      "rewards/rejected": -14.877609252929688,
      "step": 2398
    },
    {
      "epoch": 3.14,
      "learning_rate": 3.938109622406429e-05,
      "logits/chosen": -1.7378034591674805,
      "logits/rejected": -1.7265030145645142,
      "logps/chosen": -252.15228271484375,
      "logps/rejected": -382.1368408203125,
      "loss": 0.0449,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.843169689178467,
      "rewards/margins": 9.757123947143555,
      "rewards/rejected": -14.60029411315918,
      "step": 2399
    },
    {
      "epoch": 3.14,
      "learning_rate": 3.937257453115378e-05,
      "logits/chosen": -1.988635540008545,
      "logits/rejected": -1.9777966737747192,
      "logps/chosen": -213.48365783691406,
      "logps/rejected": -334.8724365234375,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.516007423400879,
      "rewards/margins": 9.487176895141602,
      "rewards/rejected": -15.003185272216797,
      "step": 2400
    },
    {
      "epoch": 3.14,
      "learning_rate": 3.9364050343122254e-05,
      "logits/chosen": -1.9818344116210938,
      "logits/rejected": -2.037508726119995,
      "logps/chosen": -187.1419677734375,
      "logps/rejected": -290.38812255859375,
      "loss": 0.0052,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.9792304039001465,
      "rewards/margins": 9.056032180786133,
      "rewards/rejected": -14.035264015197754,
      "step": 2401
    },
    {
      "epoch": 3.14,
      "learning_rate": 3.9355523661449515e-05,
      "logits/chosen": -1.9677616357803345,
      "logits/rejected": -2.0162582397460938,
      "logps/chosen": -224.53501892089844,
      "logps/rejected": -340.9477844238281,
      "loss": 0.0436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.842019557952881,
      "rewards/margins": 12.349411010742188,
      "rewards/rejected": -16.191429138183594,
      "step": 2402
    },
    {
      "epoch": 3.14,
      "learning_rate": 3.9346994487615826e-05,
      "logits/chosen": -2.144747495651245,
      "logits/rejected": -2.187472105026245,
      "logps/chosen": -257.41339111328125,
      "logps/rejected": -309.4937438964844,
      "loss": 0.0907,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.622319221496582,
      "rewards/margins": 7.701298713684082,
      "rewards/rejected": -14.323617935180664,
      "step": 2403
    },
    {
      "epoch": 3.15,
      "learning_rate": 3.933846282310187e-05,
      "logits/chosen": -2.13328218460083,
      "logits/rejected": -2.134481906890869,
      "logps/chosen": -246.9745330810547,
      "logps/rejected": -361.6680908203125,
      "loss": 0.0883,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.423940658569336,
      "rewards/margins": 9.740814208984375,
      "rewards/rejected": -16.16475486755371,
      "step": 2404
    },
    {
      "epoch": 3.15,
      "learning_rate": 3.932992866938878e-05,
      "logits/chosen": -2.129410743713379,
      "logits/rejected": -2.046457052230835,
      "logps/chosen": -315.9283752441406,
      "logps/rejected": -406.8256530761719,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.326528072357178,
      "rewards/margins": 11.082746505737305,
      "rewards/rejected": -17.40927505493164,
      "step": 2405
    },
    {
      "epoch": 3.15,
      "learning_rate": 3.9321392027958116e-05,
      "logits/chosen": -1.7999567985534668,
      "logits/rejected": -1.8146796226501465,
      "logps/chosen": -223.77041625976562,
      "logps/rejected": -356.35107421875,
      "loss": 0.0867,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.859269142150879,
      "rewards/margins": 11.880956649780273,
      "rewards/rejected": -15.740224838256836,
      "step": 2406
    },
    {
      "epoch": 3.15,
      "learning_rate": 3.931285290029184e-05,
      "logits/chosen": -2.105497360229492,
      "logits/rejected": -2.0085368156433105,
      "logps/chosen": -319.6090087890625,
      "logps/rejected": -396.0473327636719,
      "loss": 0.0869,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.423470497131348,
      "rewards/margins": 8.27132511138916,
      "rewards/rejected": -13.694795608520508,
      "step": 2407
    },
    {
      "epoch": 3.15,
      "learning_rate": 3.930431128787239e-05,
      "logits/chosen": -1.7433183193206787,
      "logits/rejected": -1.8397947549819946,
      "logps/chosen": -212.28330993652344,
      "logps/rejected": -334.6816711425781,
      "loss": 0.0873,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.340822696685791,
      "rewards/margins": 9.099462509155273,
      "rewards/rejected": -14.440284729003906,
      "step": 2408
    },
    {
      "epoch": 3.15,
      "learning_rate": 3.929576719218261e-05,
      "logits/chosen": -2.2288548946380615,
      "logits/rejected": -2.113827705383301,
      "logps/chosen": -289.6617126464844,
      "logps/rejected": -354.66961669921875,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.691329002380371,
      "rewards/margins": 9.017830848693848,
      "rewards/rejected": -15.709159851074219,
      "step": 2409
    },
    {
      "epoch": 3.15,
      "learning_rate": 3.928722061470577e-05,
      "logits/chosen": -2.0128843784332275,
      "logits/rejected": -2.063493490219116,
      "logps/chosen": -241.12109375,
      "logps/rejected": -398.9261474609375,
      "loss": 0.0435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.745203495025635,
      "rewards/margins": 12.559578895568848,
      "rewards/rejected": -18.304780960083008,
      "step": 2410
    },
    {
      "epoch": 3.16,
      "learning_rate": 3.9278671556925594e-05,
      "logits/chosen": -1.885010838508606,
      "logits/rejected": -2.011854410171509,
      "logps/chosen": -254.62257385253906,
      "logps/rejected": -302.7500915527344,
      "loss": 0.087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.52474308013916,
      "rewards/margins": 8.861490249633789,
      "rewards/rejected": -14.386234283447266,
      "step": 2411
    },
    {
      "epoch": 3.16,
      "learning_rate": 3.927012002032622e-05,
      "logits/chosen": -2.013869285583496,
      "logits/rejected": -2.0908093452453613,
      "logps/chosen": -255.79388427734375,
      "logps/rejected": -385.9837951660156,
      "loss": 0.0439,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.909495830535889,
      "rewards/margins": 8.987797737121582,
      "rewards/rejected": -15.897294044494629,
      "step": 2412
    },
    {
      "epoch": 3.16,
      "learning_rate": 3.926156600639222e-05,
      "logits/chosen": -1.8484530448913574,
      "logits/rejected": -1.924649715423584,
      "logps/chosen": -285.5309753417969,
      "logps/rejected": -396.7544860839844,
      "loss": 0.0869,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.703119277954102,
      "rewards/margins": 9.599272727966309,
      "rewards/rejected": -15.30239200592041,
      "step": 2413
    },
    {
      "epoch": 3.16,
      "learning_rate": 3.925300951660858e-05,
      "logits/chosen": -2.017725944519043,
      "logits/rejected": -2.0484378337860107,
      "logps/chosen": -249.70545959472656,
      "logps/rejected": -306.10150146484375,
      "loss": 0.0436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.9575750827789307,
      "rewards/margins": 10.65237808227539,
      "rewards/rejected": -14.609952926635742,
      "step": 2414
    },
    {
      "epoch": 3.16,
      "learning_rate": 3.924445055246076e-05,
      "logits/chosen": -2.3275504112243652,
      "logits/rejected": -2.4170687198638916,
      "logps/chosen": -232.57530212402344,
      "logps/rejected": -347.41302490234375,
      "loss": 0.1307,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.247507095336914,
      "rewards/margins": 7.428805351257324,
      "rewards/rejected": -12.676311492919922,
      "step": 2415
    },
    {
      "epoch": 3.16,
      "learning_rate": 3.923588911543461e-05,
      "logits/chosen": -1.910735845565796,
      "logits/rejected": -1.9588009119033813,
      "logps/chosen": -210.57037353515625,
      "logps/rejected": -340.1056823730469,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.114107131958008,
      "rewards/margins": 10.13116455078125,
      "rewards/rejected": -15.245270729064941,
      "step": 2416
    },
    {
      "epoch": 3.16,
      "learning_rate": 3.922732520701641e-05,
      "logits/chosen": -1.9737759828567505,
      "logits/rejected": -2.0073487758636475,
      "logps/chosen": -234.25225830078125,
      "logps/rejected": -329.5049133300781,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.450007438659668,
      "rewards/margins": 10.389273643493652,
      "rewards/rejected": -15.839282989501953,
      "step": 2417
    },
    {
      "epoch": 3.16,
      "learning_rate": 3.9218758828692885e-05,
      "logits/chosen": -1.8982748985290527,
      "logits/rejected": -2.017169713973999,
      "logps/chosen": -219.2686004638672,
      "logps/rejected": -355.2296447753906,
      "loss": 0.0871,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.760926246643066,
      "rewards/margins": 9.56212043762207,
      "rewards/rejected": -15.323046684265137,
      "step": 2418
    },
    {
      "epoch": 3.17,
      "learning_rate": 3.92101899819512e-05,
      "logits/chosen": -2.09666109085083,
      "logits/rejected": -2.1102030277252197,
      "logps/chosen": -291.5617370605469,
      "logps/rejected": -396.4189147949219,
      "loss": 0.0436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.530086994171143,
      "rewards/margins": 10.002303123474121,
      "rewards/rejected": -15.532389640808105,
      "step": 2419
    },
    {
      "epoch": 3.17,
      "learning_rate": 3.920161866827889e-05,
      "logits/chosen": -2.2040367126464844,
      "logits/rejected": -2.184612989425659,
      "logps/chosen": -201.29171752929688,
      "logps/rejected": -306.67633056640625,
      "loss": 0.1312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.141445159912109,
      "rewards/margins": 9.965995788574219,
      "rewards/rejected": -15.107441902160645,
      "step": 2420
    },
    {
      "epoch": 3.17,
      "learning_rate": 3.919304488916401e-05,
      "logits/chosen": -2.039891004562378,
      "logits/rejected": -2.046557903289795,
      "logps/chosen": -279.17388916015625,
      "logps/rejected": -368.23779296875,
      "loss": 0.0643,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.187160015106201,
      "rewards/margins": 9.28410530090332,
      "rewards/rejected": -14.47126579284668,
      "step": 2421
    },
    {
      "epoch": 3.17,
      "learning_rate": 3.918446864609496e-05,
      "logits/chosen": -1.8513938188552856,
      "logits/rejected": -1.8851006031036377,
      "logps/chosen": -244.12559509277344,
      "logps/rejected": -359.91925048828125,
      "loss": 0.0434,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.379150390625,
      "rewards/margins": 11.033041000366211,
      "rewards/rejected": -17.41219139099121,
      "step": 2422
    },
    {
      "epoch": 3.17,
      "learning_rate": 3.9175889940560626e-05,
      "logits/chosen": -2.183446168899536,
      "logits/rejected": -2.1999473571777344,
      "logps/chosen": -245.6224365234375,
      "logps/rejected": -338.06915283203125,
      "loss": 0.0463,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.378600120544434,
      "rewards/margins": 9.645833015441895,
      "rewards/rejected": -15.024433135986328,
      "step": 2423
    },
    {
      "epoch": 3.17,
      "learning_rate": 3.916730877405027e-05,
      "logits/chosen": -2.3098859786987305,
      "logits/rejected": -2.371561050415039,
      "logps/chosen": -296.7130126953125,
      "logps/rejected": -389.8238525390625,
      "loss": 0.0448,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.905816555023193,
      "rewards/margins": 10.16203784942627,
      "rewards/rejected": -17.067853927612305,
      "step": 2424
    },
    {
      "epoch": 3.17,
      "learning_rate": 3.9158725148053624e-05,
      "logits/chosen": -1.7294484376907349,
      "logits/rejected": -1.927701711654663,
      "logps/chosen": -254.4145965576172,
      "logps/rejected": -390.72906494140625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.999884605407715,
      "rewards/margins": 10.069520950317383,
      "rewards/rejected": -17.06940460205078,
      "step": 2425
    },
    {
      "epoch": 3.17,
      "learning_rate": 3.915013906406083e-05,
      "logits/chosen": -2.015231132507324,
      "logits/rejected": -1.955536961555481,
      "logps/chosen": -217.324462890625,
      "logps/rejected": -323.7929992675781,
      "loss": 0.0873,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.474260330200195,
      "rewards/margins": 8.68239974975586,
      "rewards/rejected": -14.156660079956055,
      "step": 2426
    },
    {
      "epoch": 3.18,
      "learning_rate": 3.9141550523562443e-05,
      "logits/chosen": -2.0247368812561035,
      "logits/rejected": -2.090301275253296,
      "logps/chosen": -266.1253356933594,
      "logps/rejected": -333.9754943847656,
      "loss": 0.0873,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.677989482879639,
      "rewards/margins": 8.69468879699707,
      "rewards/rejected": -15.37267780303955,
      "step": 2427
    },
    {
      "epoch": 3.18,
      "learning_rate": 3.9132959528049474e-05,
      "logits/chosen": -1.9445977210998535,
      "logits/rejected": -1.9694314002990723,
      "logps/chosen": -233.66751098632812,
      "logps/rejected": -312.2908630371094,
      "loss": 0.0878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.779075622558594,
      "rewards/margins": 8.764878273010254,
      "rewards/rejected": -14.543952941894531,
      "step": 2428
    },
    {
      "epoch": 3.18,
      "learning_rate": 3.912436607901334e-05,
      "logits/chosen": -1.892113447189331,
      "logits/rejected": -1.903243899345398,
      "logps/chosen": -230.07528686523438,
      "logps/rejected": -279.7763366699219,
      "loss": 0.0876,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.427990913391113,
      "rewards/margins": 7.037283897399902,
      "rewards/rejected": -12.465274810791016,
      "step": 2429
    },
    {
      "epoch": 3.18,
      "learning_rate": 3.91157701779459e-05,
      "logits/chosen": -2.0418152809143066,
      "logits/rejected": -2.0431063175201416,
      "logps/chosen": -289.6370849609375,
      "logps/rejected": -371.258056640625,
      "loss": 0.0869,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.584191799163818,
      "rewards/margins": 9.527920722961426,
      "rewards/rejected": -16.11211395263672,
      "step": 2430
    },
    {
      "epoch": 3.18,
      "learning_rate": 3.91071718263394e-05,
      "logits/chosen": -1.8952577114105225,
      "logits/rejected": -1.975609302520752,
      "logps/chosen": -277.6134033203125,
      "logps/rejected": -369.93914794921875,
      "loss": 0.044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.555874824523926,
      "rewards/margins": 9.395423889160156,
      "rewards/rejected": -14.951298713684082,
      "step": 2431
    },
    {
      "epoch": 3.18,
      "learning_rate": 3.909857102568657e-05,
      "logits/chosen": -1.5970344543457031,
      "logits/rejected": -1.5644360780715942,
      "logps/chosen": -290.7686462402344,
      "logps/rejected": -411.13165283203125,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.298255443572998,
      "rewards/margins": 10.077960968017578,
      "rewards/rejected": -16.376216888427734,
      "step": 2432
    },
    {
      "epoch": 3.18,
      "learning_rate": 3.908996777748051e-05,
      "logits/chosen": -1.9386467933654785,
      "logits/rejected": -2.052544593811035,
      "logps/chosen": -192.90573120117188,
      "logps/rejected": -322.02581787109375,
      "loss": 0.0435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.179001808166504,
      "rewards/margins": 10.757402420043945,
      "rewards/rejected": -14.936405181884766,
      "step": 2433
    },
    {
      "epoch": 3.19,
      "learning_rate": 3.9081362083214784e-05,
      "logits/chosen": -1.9067569971084595,
      "logits/rejected": -1.8736732006072998,
      "logps/chosen": -281.9491271972656,
      "logps/rejected": -359.25262451171875,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.667690277099609,
      "rewards/margins": 9.953161239624023,
      "rewards/rejected": -15.620849609375,
      "step": 2434
    },
    {
      "epoch": 3.19,
      "learning_rate": 3.907275394438335e-05,
      "logits/chosen": -2.115126609802246,
      "logits/rejected": -2.1041605472564697,
      "logps/chosen": -256.5445251464844,
      "logps/rejected": -362.7337646484375,
      "loss": 0.0436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.392908096313477,
      "rewards/margins": 9.380439758300781,
      "rewards/rejected": -14.773348808288574,
      "step": 2435
    },
    {
      "epoch": 3.19,
      "learning_rate": 3.9064143362480637e-05,
      "logits/chosen": -1.8047089576721191,
      "logits/rejected": -1.767399787902832,
      "logps/chosen": -253.64195251464844,
      "logps/rejected": -313.3564147949219,
      "loss": 0.0494,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.76961088180542,
      "rewards/margins": 6.63554573059082,
      "rewards/rejected": -12.405157089233398,
      "step": 2436
    },
    {
      "epoch": 3.19,
      "learning_rate": 3.905553033900144e-05,
      "logits/chosen": -2.1431894302368164,
      "logits/rejected": -2.1688289642333984,
      "logps/chosen": -202.8959503173828,
      "logps/rejected": -318.6661376953125,
      "loss": 0.1241,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.242168426513672,
      "rewards/margins": 9.04034423828125,
      "rewards/rejected": -14.282512664794922,
      "step": 2437
    },
    {
      "epoch": 3.19,
      "learning_rate": 3.904691487544101e-05,
      "logits/chosen": -1.6103119850158691,
      "logits/rejected": -1.6657888889312744,
      "logps/chosen": -312.8416748046875,
      "logps/rejected": -410.4940185546875,
      "loss": 0.087,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.678059101104736,
      "rewards/margins": 9.324478149414062,
      "rewards/rejected": -17.00253677368164,
      "step": 2438
    },
    {
      "epoch": 3.19,
      "learning_rate": 3.903829697329502e-05,
      "logits/chosen": -2.0904810428619385,
      "logits/rejected": -2.199312448501587,
      "logps/chosen": -237.72813415527344,
      "logps/rejected": -360.5721130371094,
      "loss": 0.0435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.821403503417969,
      "rewards/margins": 9.57381534576416,
      "rewards/rejected": -15.395220756530762,
      "step": 2439
    },
    {
      "epoch": 3.19,
      "learning_rate": 3.902967663405956e-05,
      "logits/chosen": -2.03212308883667,
      "logits/rejected": -2.1629748344421387,
      "logps/chosen": -267.4989929199219,
      "logps/rejected": -342.18365478515625,
      "loss": 0.0869,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.316411018371582,
      "rewards/margins": 7.876006126403809,
      "rewards/rejected": -13.19241714477539,
      "step": 2440
    },
    {
      "epoch": 3.19,
      "learning_rate": 3.902105385923115e-05,
      "logits/chosen": -1.9315513372421265,
      "logits/rejected": -1.8531064987182617,
      "logps/chosen": -275.96307373046875,
      "logps/rejected": -345.4717712402344,
      "loss": 0.1735,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -5.740131378173828,
      "rewards/margins": 7.676638603210449,
      "rewards/rejected": -13.416770935058594,
      "step": 2441
    },
    {
      "epoch": 3.2,
      "learning_rate": 3.901242865030673e-05,
      "logits/chosen": -1.7603967189788818,
      "logits/rejected": -1.808661699295044,
      "logps/chosen": -171.9634552001953,
      "logps/rejected": -302.039794921875,
      "loss": 0.0437,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.7204270362854,
      "rewards/margins": 9.628297805786133,
      "rewards/rejected": -14.348724365234375,
      "step": 2442
    },
    {
      "epoch": 3.2,
      "learning_rate": 3.900380100878366e-05,
      "logits/chosen": -1.9438949823379517,
      "logits/rejected": -1.9663145542144775,
      "logps/chosen": -211.74940490722656,
      "logps/rejected": -284.43328857421875,
      "loss": 0.1803,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.087337970733643,
      "rewards/margins": 6.456294059753418,
      "rewards/rejected": -11.543631553649902,
      "step": 2443
    },
    {
      "epoch": 3.2,
      "learning_rate": 3.899517093615971e-05,
      "logits/chosen": -1.6884608268737793,
      "logits/rejected": -1.7831130027770996,
      "logps/chosen": -256.3289489746094,
      "logps/rejected": -407.73321533203125,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.849040508270264,
      "rewards/margins": 11.37957763671875,
      "rewards/rejected": -17.228618621826172,
      "step": 2444
    },
    {
      "epoch": 3.2,
      "learning_rate": 3.8986538433933116e-05,
      "logits/chosen": -2.0082106590270996,
      "logits/rejected": -2.0837900638580322,
      "logps/chosen": -252.9528045654297,
      "logps/rejected": -404.23822021484375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.35462760925293,
      "rewards/margins": 11.114782333374023,
      "rewards/rejected": -17.469409942626953,
      "step": 2445
    },
    {
      "epoch": 3.2,
      "learning_rate": 3.8977903503602476e-05,
      "logits/chosen": -1.9969182014465332,
      "logits/rejected": -2.0156984329223633,
      "logps/chosen": -231.7480926513672,
      "logps/rejected": -348.5244445800781,
      "loss": 0.168,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.384876251220703,
      "rewards/margins": 7.84205436706543,
      "rewards/rejected": -12.22693157196045,
      "step": 2446
    },
    {
      "epoch": 3.2,
      "learning_rate": 3.896926614666686e-05,
      "logits/chosen": -1.9319980144500732,
      "logits/rejected": -1.9487107992172241,
      "logps/chosen": -269.6474609375,
      "logps/rejected": -402.9222412109375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.144097328186035,
      "rewards/margins": 10.469097137451172,
      "rewards/rejected": -17.61319351196289,
      "step": 2447
    },
    {
      "epoch": 3.2,
      "learning_rate": 3.896062636462572e-05,
      "logits/chosen": -1.6876417398452759,
      "logits/rejected": -1.7451732158660889,
      "logps/chosen": -255.68814086914062,
      "logps/rejected": -343.18280029296875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.484189987182617,
      "rewards/margins": 9.643763542175293,
      "rewards/rejected": -17.127952575683594,
      "step": 2448
    },
    {
      "epoch": 3.2,
      "learning_rate": 3.895198415897896e-05,
      "logits/chosen": -2.1559417247772217,
      "logits/rejected": -2.2037086486816406,
      "logps/chosen": -244.03125,
      "logps/rejected": -376.2003173828125,
      "loss": 0.0869,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.5298991203308105,
      "rewards/margins": 8.450559616088867,
      "rewards/rejected": -14.980459213256836,
      "step": 2449
    },
    {
      "epoch": 3.21,
      "learning_rate": 3.8943339531226895e-05,
      "logits/chosen": -2.036494255065918,
      "logits/rejected": -2.1403274536132812,
      "logps/chosen": -238.76034545898438,
      "logps/rejected": -350.3614196777344,
      "loss": 0.0051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.96282958984375,
      "rewards/margins": 9.401100158691406,
      "rewards/rejected": -15.36392879486084,
      "step": 2450
    },
    {
      "epoch": 3.21,
      "learning_rate": 3.893469248287025e-05,
      "logits/chosen": -2.0279226303100586,
      "logits/rejected": -2.021245241165161,
      "logps/chosen": -231.94520568847656,
      "logps/rejected": -349.776123046875,
      "loss": 0.0435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.093944549560547,
      "rewards/margins": 11.85527229309082,
      "rewards/rejected": -16.949216842651367,
      "step": 2451
    },
    {
      "epoch": 3.21,
      "learning_rate": 3.892604301541018e-05,
      "logits/chosen": -1.7343417406082153,
      "logits/rejected": -1.8236770629882812,
      "logps/chosen": -270.17864990234375,
      "logps/rejected": -362.18426513671875,
      "loss": 0.0868,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -7.230927467346191,
      "rewards/margins": 8.540557861328125,
      "rewards/rejected": -15.77148723602295,
      "step": 2452
    },
    {
      "epoch": 3.21,
      "learning_rate": 3.891739113034826e-05,
      "logits/chosen": -2.024186849594116,
      "logits/rejected": -2.0081472396850586,
      "logps/chosen": -245.2545166015625,
      "logps/rejected": -332.8877868652344,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.378724575042725,
      "rewards/margins": 10.643712997436523,
      "rewards/rejected": -16.022436141967773,
      "step": 2453
    },
    {
      "epoch": 3.21,
      "learning_rate": 3.890873682918648e-05,
      "logits/chosen": -1.7880804538726807,
      "logits/rejected": -1.9576160907745361,
      "logps/chosen": -184.98348999023438,
      "logps/rejected": -287.5200500488281,
      "loss": 0.0896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.778811454772949,
      "rewards/margins": 8.423736572265625,
      "rewards/rejected": -13.202548027038574,
      "step": 2454
    },
    {
      "epoch": 3.21,
      "learning_rate": 3.890008011342725e-05,
      "logits/chosen": -1.8650615215301514,
      "logits/rejected": -1.9369796514511108,
      "logps/chosen": -280.738525390625,
      "logps/rejected": -404.7315979003906,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.032692909240723,
      "rewards/margins": 11.05531120300293,
      "rewards/rejected": -17.088003158569336,
      "step": 2455
    },
    {
      "epoch": 3.21,
      "learning_rate": 3.88914209845734e-05,
      "logits/chosen": -2.0854010581970215,
      "logits/rejected": -2.2006287574768066,
      "logps/chosen": -240.37437438964844,
      "logps/rejected": -410.825439453125,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.425436019897461,
      "rewards/margins": 10.704320907592773,
      "rewards/rejected": -17.129756927490234,
      "step": 2456
    },
    {
      "epoch": 3.22,
      "learning_rate": 3.888275944412819e-05,
      "logits/chosen": -1.9605014324188232,
      "logits/rejected": -1.9867416620254517,
      "logps/chosen": -267.28179931640625,
      "logps/rejected": -380.3411560058594,
      "loss": 0.045,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.87191104888916,
      "rewards/margins": 9.627467155456543,
      "rewards/rejected": -16.499380111694336,
      "step": 2457
    },
    {
      "epoch": 3.22,
      "learning_rate": 3.887409549359528e-05,
      "logits/chosen": -2.3427610397338867,
      "logits/rejected": -2.385375738143921,
      "logps/chosen": -221.71961975097656,
      "logps/rejected": -319.4991455078125,
      "loss": 0.045,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.360358238220215,
      "rewards/margins": 8.489831924438477,
      "rewards/rejected": -13.850191116333008,
      "step": 2458
    },
    {
      "epoch": 3.22,
      "learning_rate": 3.886542913447876e-05,
      "logits/chosen": -2.1394031047821045,
      "logits/rejected": -2.1759281158447266,
      "logps/chosen": -254.70877075195312,
      "logps/rejected": -348.2254943847656,
      "loss": 0.0437,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.5420122146606445,
      "rewards/margins": 10.061568260192871,
      "rewards/rejected": -15.603580474853516,
      "step": 2459
    },
    {
      "epoch": 3.22,
      "learning_rate": 3.8856760368283143e-05,
      "logits/chosen": -1.9917089939117432,
      "logits/rejected": -1.995586633682251,
      "logps/chosen": -245.02655029296875,
      "logps/rejected": -338.9930725097656,
      "loss": 0.0436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.434337139129639,
      "rewards/margins": 9.70940113067627,
      "rewards/rejected": -15.143738746643066,
      "step": 2460
    },
    {
      "epoch": 3.22,
      "learning_rate": 3.884808919651334e-05,
      "logits/chosen": -2.125701665878296,
      "logits/rejected": -2.1440436840057373,
      "logps/chosen": -251.43545532226562,
      "logps/rejected": -355.54449462890625,
      "loss": 0.0441,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.037803649902344,
      "rewards/margins": 10.139799118041992,
      "rewards/rejected": -16.177602767944336,
      "step": 2461
    },
    {
      "epoch": 3.22,
      "learning_rate": 3.883941562067469e-05,
      "logits/chosen": -1.7071962356567383,
      "logits/rejected": -1.725158929824829,
      "logps/chosen": -230.553466796875,
      "logps/rejected": -310.40972900390625,
      "loss": 0.0445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.98706579208374,
      "rewards/margins": 8.925557136535645,
      "rewards/rejected": -14.912622451782227,
      "step": 2462
    },
    {
      "epoch": 3.22,
      "learning_rate": 3.8830739642272965e-05,
      "logits/chosen": -2.2549314498901367,
      "logits/rejected": -2.256117105484009,
      "logps/chosen": -252.87001037597656,
      "logps/rejected": -331.2015380859375,
      "loss": 0.0443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.360776424407959,
      "rewards/margins": 9.707989692687988,
      "rewards/rejected": -15.068767547607422,
      "step": 2463
    },
    {
      "epoch": 3.22,
      "learning_rate": 3.882206126281432e-05,
      "logits/chosen": -1.7178093194961548,
      "logits/rejected": -1.7864190340042114,
      "logps/chosen": -261.02215576171875,
      "logps/rejected": -351.16302490234375,
      "loss": 0.0872,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.090516567230225,
      "rewards/margins": 8.126142501831055,
      "rewards/rejected": -14.216659545898438,
      "step": 2464
    },
    {
      "epoch": 3.23,
      "learning_rate": 3.8813380483805374e-05,
      "logits/chosen": -2.1009628772735596,
      "logits/rejected": -2.0164217948913574,
      "logps/chosen": -262.838134765625,
      "logps/rejected": -359.9693603515625,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.574399948120117,
      "rewards/margins": 11.433204650878906,
      "rewards/rejected": -17.00760269165039,
      "step": 2465
    },
    {
      "epoch": 3.23,
      "learning_rate": 3.8804697306753114e-05,
      "logits/chosen": -2.1591877937316895,
      "logits/rejected": -2.1212711334228516,
      "logps/chosen": -222.40719604492188,
      "logps/rejected": -327.9382019042969,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.407850742340088,
      "rewards/margins": 9.4583101272583,
      "rewards/rejected": -14.866161346435547,
      "step": 2466
    },
    {
      "epoch": 3.23,
      "learning_rate": 3.879601173316498e-05,
      "logits/chosen": -1.9027129411697388,
      "logits/rejected": -1.9284546375274658,
      "logps/chosen": -216.7272186279297,
      "logps/rejected": -337.1361083984375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.403012752532959,
      "rewards/margins": 12.751434326171875,
      "rewards/rejected": -17.154449462890625,
      "step": 2467
    },
    {
      "epoch": 3.23,
      "learning_rate": 3.87873237645488e-05,
      "logits/chosen": -1.7814953327178955,
      "logits/rejected": -1.8257054090499878,
      "logps/chosen": -225.2574462890625,
      "logps/rejected": -346.04974365234375,
      "loss": 0.0044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.834550380706787,
      "rewards/margins": 10.236806869506836,
      "rewards/rejected": -16.07135772705078,
      "step": 2468
    },
    {
      "epoch": 3.23,
      "learning_rate": 3.8778633402412836e-05,
      "logits/chosen": -2.073378324508667,
      "logits/rejected": -2.1560871601104736,
      "logps/chosen": -213.5805206298828,
      "logps/rejected": -301.17755126953125,
      "loss": 0.0031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.958551406860352,
      "rewards/margins": 9.559164047241211,
      "rewards/rejected": -14.517715454101562,
      "step": 2469
    },
    {
      "epoch": 3.23,
      "learning_rate": 3.876994064826576e-05,
      "logits/chosen": -2.0379161834716797,
      "logits/rejected": -2.056102752685547,
      "logps/chosen": -275.51348876953125,
      "logps/rejected": -376.2273864746094,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.516529560089111,
      "rewards/margins": 10.124565124511719,
      "rewards/rejected": -16.641094207763672,
      "step": 2470
    },
    {
      "epoch": 3.23,
      "learning_rate": 3.876124550361667e-05,
      "logits/chosen": -2.012174129486084,
      "logits/rejected": -1.9914698600769043,
      "logps/chosen": -271.0264587402344,
      "logps/rejected": -371.2334289550781,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.945853233337402,
      "rewards/margins": 10.607396125793457,
      "rewards/rejected": -17.55324935913086,
      "step": 2471
    },
    {
      "epoch": 3.24,
      "learning_rate": 3.875254796997505e-05,
      "logits/chosen": -1.8690792322158813,
      "logits/rejected": -1.9150246381759644,
      "logps/chosen": -281.6956787109375,
      "logps/rejected": -394.4105224609375,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.801252365112305,
      "rewards/margins": 10.3662748336792,
      "rewards/rejected": -17.16752815246582,
      "step": 2472
    },
    {
      "epoch": 3.24,
      "learning_rate": 3.874384804885082e-05,
      "logits/chosen": -1.8989354372024536,
      "logits/rejected": -1.9007232189178467,
      "logps/chosen": -249.84799194335938,
      "logps/rejected": -324.7830505371094,
      "loss": 0.0439,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.7783098220825195,
      "rewards/margins": 8.47901725769043,
      "rewards/rejected": -14.257326126098633,
      "step": 2473
    },
    {
      "epoch": 3.24,
      "learning_rate": 3.873514574175433e-05,
      "logits/chosen": -1.9035624265670776,
      "logits/rejected": -1.9880552291870117,
      "logps/chosen": -225.6424560546875,
      "logps/rejected": -357.0283203125,
      "loss": 0.0868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.232097625732422,
      "rewards/margins": 9.460615158081055,
      "rewards/rejected": -15.69271183013916,
      "step": 2474
    },
    {
      "epoch": 3.24,
      "learning_rate": 3.8726441050196305e-05,
      "logits/chosen": -2.067682981491089,
      "logits/rejected": -2.197803258895874,
      "logps/chosen": -312.9625549316406,
      "logps/rejected": -411.094970703125,
      "loss": 0.0436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.463919639587402,
      "rewards/margins": 10.368206977844238,
      "rewards/rejected": -16.832128524780273,
      "step": 2475
    },
    {
      "epoch": 3.24,
      "learning_rate": 3.871773397568792e-05,
      "logits/chosen": -1.8097530603408813,
      "logits/rejected": -1.8437132835388184,
      "logps/chosen": -245.35177612304688,
      "logps/rejected": -360.3444519042969,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.219241619110107,
      "rewards/margins": 10.346756935119629,
      "rewards/rejected": -15.565999031066895,
      "step": 2476
    },
    {
      "epoch": 3.24,
      "learning_rate": 3.8709024519740735e-05,
      "logits/chosen": -1.8741097450256348,
      "logits/rejected": -1.864046335220337,
      "logps/chosen": -236.70962524414062,
      "logps/rejected": -333.8223571777344,
      "loss": 0.0435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.540530681610107,
      "rewards/margins": 10.588478088378906,
      "rewards/rejected": -15.129007339477539,
      "step": 2477
    },
    {
      "epoch": 3.24,
      "learning_rate": 3.870031268386676e-05,
      "logits/chosen": -2.0998189449310303,
      "logits/rejected": -2.161912202835083,
      "logps/chosen": -253.3290557861328,
      "logps/rejected": -377.12567138671875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.067862510681152,
      "rewards/margins": 10.710309982299805,
      "rewards/rejected": -16.77817153930664,
      "step": 2478
    },
    {
      "epoch": 3.24,
      "learning_rate": 3.869159846957836e-05,
      "logits/chosen": -1.9125173091888428,
      "logits/rejected": -1.9453988075256348,
      "logps/chosen": -246.0362548828125,
      "logps/rejected": -325.9284973144531,
      "loss": 0.0871,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.746542930603027,
      "rewards/margins": 9.125516891479492,
      "rewards/rejected": -14.872061729431152,
      "step": 2479
    },
    {
      "epoch": 3.25,
      "learning_rate": 3.8682881878388376e-05,
      "logits/chosen": -1.9923452138900757,
      "logits/rejected": -2.0072903633117676,
      "logps/chosen": -277.7795104980469,
      "logps/rejected": -377.7403564453125,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.331365585327148,
      "rewards/margins": 10.37812614440918,
      "rewards/rejected": -16.709491729736328,
      "step": 2480
    },
    {
      "epoch": 3.25,
      "learning_rate": 3.867416291181003e-05,
      "logits/chosen": -2.1541683673858643,
      "logits/rejected": -2.184211492538452,
      "logps/chosen": -259.4024658203125,
      "logps/rejected": -336.68927001953125,
      "loss": 0.0078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.420302391052246,
      "rewards/margins": 8.163504600524902,
      "rewards/rejected": -15.583806991577148,
      "step": 2481
    },
    {
      "epoch": 3.25,
      "learning_rate": 3.866544157135695e-05,
      "logits/chosen": -2.1215081214904785,
      "logits/rejected": -2.0577147006988525,
      "logps/chosen": -278.74676513671875,
      "logps/rejected": -354.3476867675781,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.952124118804932,
      "rewards/margins": 9.676928520202637,
      "rewards/rejected": -15.629051208496094,
      "step": 2482
    },
    {
      "epoch": 3.25,
      "learning_rate": 3.8656717858543195e-05,
      "logits/chosen": -1.8749539852142334,
      "logits/rejected": -1.8486679792404175,
      "logps/chosen": -215.23916625976562,
      "logps/rejected": -323.09613037109375,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.153021335601807,
      "rewards/margins": 9.829922676086426,
      "rewards/rejected": -15.98294448852539,
      "step": 2483
    },
    {
      "epoch": 3.25,
      "learning_rate": 3.864799177488322e-05,
      "logits/chosen": -2.078026294708252,
      "logits/rejected": -2.025261402130127,
      "logps/chosen": -285.80206298828125,
      "logps/rejected": -350.468017578125,
      "loss": 0.0498,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.340088844299316,
      "rewards/margins": 8.425580024719238,
      "rewards/rejected": -14.765668869018555,
      "step": 2484
    },
    {
      "epoch": 3.25,
      "learning_rate": 3.863926332189191e-05,
      "logits/chosen": -1.8989228010177612,
      "logits/rejected": -1.887642502784729,
      "logps/chosen": -209.74522399902344,
      "logps/rejected": -319.71978759765625,
      "loss": 0.0868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.254289150238037,
      "rewards/margins": 9.678909301757812,
      "rewards/rejected": -14.933197975158691,
      "step": 2485
    },
    {
      "epoch": 3.25,
      "learning_rate": 3.8630532501084535e-05,
      "logits/chosen": -1.9497569799423218,
      "logits/rejected": -2.0822038650512695,
      "logps/chosen": -165.8619842529297,
      "logps/rejected": -305.4637451171875,
      "loss": 0.1316,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -4.822153091430664,
      "rewards/margins": 9.318231582641602,
      "rewards/rejected": -14.140385627746582,
      "step": 2486
    },
    {
      "epoch": 3.25,
      "learning_rate": 3.8621799313976805e-05,
      "logits/chosen": -1.9201487302780151,
      "logits/rejected": -1.9919875860214233,
      "logps/chosen": -236.57803344726562,
      "logps/rejected": -345.8396301269531,
      "loss": 0.0131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.3776373863220215,
      "rewards/margins": 10.207651138305664,
      "rewards/rejected": -16.585289001464844,
      "step": 2487
    },
    {
      "epoch": 3.26,
      "learning_rate": 3.861306376208481e-05,
      "logits/chosen": -2.135756492614746,
      "logits/rejected": -2.178420305252075,
      "logps/chosen": -276.371337890625,
      "logps/rejected": -428.98321533203125,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -7.153833866119385,
      "rewards/margins": 11.597715377807617,
      "rewards/rejected": -18.751548767089844,
      "step": 2488
    },
    {
      "epoch": 3.26,
      "learning_rate": 3.860432584692509e-05,
      "logits/chosen": -1.9746434688568115,
      "logits/rejected": -2.049467086791992,
      "logps/chosen": -244.74005126953125,
      "logps/rejected": -384.7591247558594,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.734624862670898,
      "rewards/margins": 13.104833602905273,
      "rewards/rejected": -18.839460372924805,
      "step": 2489
    },
    {
      "epoch": 3.26,
      "learning_rate": 3.859558557001456e-05,
      "logits/chosen": -1.8276326656341553,
      "logits/rejected": -1.8128743171691895,
      "logps/chosen": -242.62420654296875,
      "logps/rejected": -370.6613464355469,
      "loss": 0.0435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.021103858947754,
      "rewards/margins": 11.176106452941895,
      "rewards/rejected": -17.19721031188965,
      "step": 2490
    },
    {
      "epoch": 3.26,
      "learning_rate": 3.858684293287055e-05,
      "logits/chosen": -1.827390193939209,
      "logits/rejected": -1.8298470973968506,
      "logps/chosen": -314.5787048339844,
      "logps/rejected": -394.1010437011719,
      "loss": 0.0868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -7.975650787353516,
      "rewards/margins": 8.666006088256836,
      "rewards/rejected": -16.641658782958984,
      "step": 2491
    },
    {
      "epoch": 3.26,
      "learning_rate": 3.857809793701082e-05,
      "logits/chosen": -1.9679362773895264,
      "logits/rejected": -1.9574182033538818,
      "logps/chosen": -205.41207885742188,
      "logps/rejected": -321.79034423828125,
      "loss": 0.1353,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -3.5130960941314697,
      "rewards/margins": 10.656757354736328,
      "rewards/rejected": -14.169854164123535,
      "step": 2492
    },
    {
      "epoch": 3.26,
      "learning_rate": 3.856935058395352e-05,
      "logits/chosen": -1.8985028266906738,
      "logits/rejected": -1.9866564273834229,
      "logps/chosen": -231.4768524169922,
      "logps/rejected": -359.1987609863281,
      "loss": 0.0868,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.280361652374268,
      "rewards/margins": 8.678674697875977,
      "rewards/rejected": -14.959036827087402,
      "step": 2493
    },
    {
      "epoch": 3.26,
      "learning_rate": 3.8560600875217235e-05,
      "logits/chosen": -1.9721897840499878,
      "logits/rejected": -2.0196175575256348,
      "logps/chosen": -184.3533172607422,
      "logps/rejected": -299.4060974121094,
      "loss": 0.1734,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -5.074180603027344,
      "rewards/margins": 9.483386039733887,
      "rewards/rejected": -14.55756664276123,
      "step": 2494
    },
    {
      "epoch": 3.27,
      "learning_rate": 3.8551848812320904e-05,
      "logits/chosen": -1.945467233657837,
      "logits/rejected": -1.9828685522079468,
      "logps/chosen": -295.1348571777344,
      "logps/rejected": -422.5150451660156,
      "loss": 0.0877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.004589080810547,
      "rewards/margins": 10.746018409729004,
      "rewards/rejected": -15.750608444213867,
      "step": 2495
    },
    {
      "epoch": 3.27,
      "learning_rate": 3.854309439678394e-05,
      "logits/chosen": -1.9645689725875854,
      "logits/rejected": -1.9029303789138794,
      "logps/chosen": -236.8740692138672,
      "logps/rejected": -359.9755554199219,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.681175708770752,
      "rewards/margins": 11.418691635131836,
      "rewards/rejected": -18.099868774414062,
      "step": 2496
    },
    {
      "epoch": 3.27,
      "learning_rate": 3.853433763012612e-05,
      "logits/chosen": -2.2054662704467773,
      "logits/rejected": -2.1916756629943848,
      "logps/chosen": -250.4889373779297,
      "logps/rejected": -302.42645263671875,
      "loss": 0.0927,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.19832706451416,
      "rewards/margins": 9.23891830444336,
      "rewards/rejected": -14.437244415283203,
      "step": 2497
    },
    {
      "epoch": 3.27,
      "learning_rate": 3.852557851386766e-05,
      "logits/chosen": -2.000141143798828,
      "logits/rejected": -1.994922399520874,
      "logps/chosen": -203.5865478515625,
      "logps/rejected": -293.255615234375,
      "loss": 0.1736,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -5.755192756652832,
      "rewards/margins": 7.394274711608887,
      "rewards/rejected": -13.149467468261719,
      "step": 2498
    },
    {
      "epoch": 3.27,
      "learning_rate": 3.8516817049529144e-05,
      "logits/chosen": -1.904963731765747,
      "logits/rejected": -1.9929938316345215,
      "logps/chosen": -228.28164672851562,
      "logps/rejected": -359.9959411621094,
      "loss": 0.0536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -6.353829383850098,
      "rewards/margins": 9.504854202270508,
      "rewards/rejected": -15.858683586120605,
      "step": 2499
    },
    {
      "epoch": 3.27,
      "learning_rate": 3.8508053238631614e-05,
      "logits/chosen": -1.7504050731658936,
      "logits/rejected": -1.6863926649093628,
      "logps/chosen": -251.31768798828125,
      "logps/rejected": -340.1594543457031,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -5.837611675262451,
      "rewards/margins": 9.95163631439209,
      "rewards/rejected": -15.7892484664917,
      "step": 2500
    }
  ],
  "logging_steps": 1,
  "max_steps": 7640,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 10,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}