Upload folder using huggingface_hub

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "../educlassifier/checkpoint-400",
+  "_name_or_path": "pL-Community/GermanEduScorer-Qwen2-1.5b",
   "architectures": [
     "Qwen2ForCausalLM"
   ],

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be9bdf8f55eb810bab80529acd87e4b1c9c9d3474842f45b1dee3b1a49014da7
+oid sha256:880b5f6e44d430c0e4e5f3fcb8956cb73110d70521dec67489aaec866e4e7d57
 size 3087467144

trainer_state.json

@@ -1,359 +1,869 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.392349190779794,
+  "epoch": 0.196174595389897,
   "eval_steps": 5000,
-  "global_step": 200,
+  "global_step": 50,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
+    {
+      "epoch": 0.00392349190779794,
+      "grad_norm": 1.34375,
+      "learning_rate": 1.153846153846154e-06,
+      "logits/chosen": -2.9852147102355957,
+      "logits/rejected": -2.9852118492126465,
+      "logps/chosen": -0.2645491361618042,
+      "logps/rejected": -3.2006759643554688,
+      "loss": 0.2782,
+      "odds_ratio_loss": 1.0742381811141968,
+      "rewards/accuracies": 0.953125,
+      "rewards/chosen": -0.02645491622388363,
+      "rewards/margins": 0.29361265897750854,
+      "rewards/rejected": -0.3200675845146179,
+      "sft_loss": 0.17080938816070557,
+      "step": 1
+    },
+    {
+      "epoch": 0.00784698381559588,
+      "grad_norm": 1.390625,
+      "learning_rate": 2.307692307692308e-06,
+      "logits/chosen": -2.9859261512756348,
+      "logits/rejected": -2.9859182834625244,
+      "logps/chosen": -0.2948899567127228,
+      "logps/rejected": -3.0603623390197754,
+      "loss": 0.3098,
+      "odds_ratio_loss": 1.081099033355713,
+      "rewards/accuracies": 0.947265625,
+      "rewards/chosen": -0.029488995671272278,
+      "rewards/margins": 0.27654725313186646,
+      "rewards/rejected": -0.30603623390197754,
+      "sft_loss": 0.20172670483589172,
+      "step": 2
+    },
+    {
+      "epoch": 0.011770475723393821,
+      "grad_norm": 1.6484375,
+      "learning_rate": 3.4615384615384617e-06,
+      "logits/chosen": -2.9848945140838623,
+      "logits/rejected": -2.984874963760376,
+      "logps/chosen": -0.30154696106910706,
+      "logps/rejected": -3.0100128650665283,
+      "loss": 0.3208,
+      "odds_ratio_loss": 1.2148220539093018,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.030154697597026825,
+      "rewards/margins": 0.27084657549858093,
+      "rewards/rejected": -0.30100125074386597,
+      "sft_loss": 0.1993543654680252,
+      "step": 3
+    },
+    {
+      "epoch": 0.01569396763119176,
+      "grad_norm": 1.453125,
+      "learning_rate": 4.615384615384616e-06,
+      "logits/chosen": -2.983187675476074,
+      "logits/rejected": -2.983184576034546,
+      "logps/chosen": -0.24826335906982422,
+      "logps/rejected": -3.0838959217071533,
+      "loss": 0.2632,
+      "odds_ratio_loss": 1.0045406818389893,
+      "rewards/accuracies": 0.9443359375,
+      "rewards/chosen": -0.02482633665204048,
+      "rewards/margins": 0.2835632264614105,
+      "rewards/rejected": -0.3083895742893219,
+      "sft_loss": 0.16278076171875,
+      "step": 4
+    },
     {
       "epoch": 0.0196174595389897,
-      "grad_norm": 2.734375,
-      "learning_rate": 5.88235294117647e-05,
-      "logits/chosen": -3.38761830329895,
-      "logits/rejected": -3.3876290321350098,
-      "logps/chosen": -0.33322039246559143,
-      "logps/rejected": -2.6748645305633545,
-      "loss": 0.3505,
-      "odds_ratio_loss": 1.2291936874389648,
-      "rewards/accuracies": 0.9292968511581421,
-      "rewards/chosen": -0.0333220399916172,
-      "rewards/margins": 0.23416443169116974,
-      "rewards/rejected": -0.26748648285865784,
-      "sft_loss": 0.22762596607208252,
-      "step": 10
+      "grad_norm": 1.2890625,
+      "learning_rate": 5.76923076923077e-06,
+      "logits/chosen": -2.9843859672546387,
+      "logits/rejected": -2.9843790531158447,
+      "logps/chosen": -0.26491791009902954,
+      "logps/rejected": -3.030116081237793,
+      "loss": 0.2815,
+      "odds_ratio_loss": 1.1153428554534912,
+      "rewards/accuracies": 0.939453125,
+      "rewards/chosen": -0.026491792872548103,
+      "rewards/margins": 0.276519775390625,
+      "rewards/rejected": -0.30301156640052795,
+      "sft_loss": 0.16996388137340546,
+      "step": 5
+    },
+    {
+      "epoch": 0.023540951446787643,
+      "grad_norm": 0.90234375,
+      "learning_rate": 6.923076923076923e-06,
+      "logits/chosen": -2.9830667972564697,
+      "logits/rejected": -2.9830613136291504,
+      "logps/chosen": -0.25957372784614563,
+      "logps/rejected": -3.0386571884155273,
+      "loss": 0.2731,
+      "odds_ratio_loss": 1.0385775566101074,
+      "rewards/accuracies": 0.9404296875,
+      "rewards/chosen": -0.025957372039556503,
+      "rewards/margins": 0.2779083251953125,
+      "rewards/rejected": -0.3038657009601593,
+      "sft_loss": 0.16922664642333984,
+      "step": 6
+    },
+    {
+      "epoch": 0.02746444335458558,
+      "grad_norm": 1.2421875,
+      "learning_rate": 8.076923076923077e-06,
+      "logits/chosen": -2.9781432151794434,
+      "logits/rejected": -2.9781363010406494,
+      "logps/chosen": -0.24075695872306824,
+      "logps/rejected": -3.1340367794036865,
+      "loss": 0.2557,
+      "odds_ratio_loss": 1.0184863805770874,
+      "rewards/accuracies": 0.9462890625,
+      "rewards/chosen": -0.024075698107481003,
+      "rewards/margins": 0.289328008890152,
+      "rewards/rejected": -0.3134036958217621,
+      "sft_loss": 0.15385302901268005,
+      "step": 7
+    },
+    {
+      "epoch": 0.03138793526238352,
+      "grad_norm": 0.84765625,
+      "learning_rate": 9.230769230769232e-06,
+      "logits/chosen": -2.9820947647094727,
+      "logits/rejected": -2.982091188430786,
+      "logps/chosen": -0.2752811908721924,
+      "logps/rejected": -3.0845463275909424,
+      "loss": 0.2907,
+      "odds_ratio_loss": 1.0758346319198608,
+      "rewards/accuracies": 0.9384765625,
+      "rewards/chosen": -0.02752811834216118,
+      "rewards/margins": 0.28092649579048157,
+      "rewards/rejected": -0.30845463275909424,
+      "sft_loss": 0.18306681513786316,
+      "step": 8
+    },
+    {
+      "epoch": 0.03531142717018146,
+      "grad_norm": 1.109375,
+      "learning_rate": 1.0384615384615384e-05,
+      "logits/chosen": -2.980917453765869,
+      "logits/rejected": -2.9809200763702393,
+      "logps/chosen": -0.25007593631744385,
+      "logps/rejected": -3.1210172176361084,
+      "loss": 0.2663,
+      "odds_ratio_loss": 0.9918112754821777,
+      "rewards/accuracies": 0.9404296875,
+      "rewards/chosen": -0.025007594376802444,
+      "rewards/margins": 0.2870941162109375,
+      "rewards/rejected": -0.31210172176361084,
+      "sft_loss": 0.16709110140800476,
+      "step": 9
     },
     {
       "epoch": 0.0392349190779794,
-      "grad_norm": 2.40625,
-      "learning_rate": 0.0001176470588235294,
-      "logits/chosen": -3.392183303833008,
-      "logits/rejected": -3.392207622528076,
-      "logps/chosen": -0.2998594641685486,
-      "logps/rejected": -3.382711887359619,
-      "loss": 0.3159,
-      "odds_ratio_loss": 1.1517291069030762,
-      "rewards/accuracies": 0.9417968988418579,
-      "rewards/chosen": -0.0299859456717968,
-      "rewards/margins": 0.3082852363586426,
-      "rewards/rejected": -0.33827120065689087,
-      "sft_loss": 0.20074963569641113,
-      "step": 20
+      "grad_norm": 0.98828125,
+      "learning_rate": 1.153846153846154e-05,
+      "logits/chosen": -2.976473569869995,
+      "logits/rejected": -2.976468086242676,
+      "logps/chosen": -0.23895932734012604,
+      "logps/rejected": -3.23227596282959,
+      "loss": 0.2498,
+      "odds_ratio_loss": 0.9731241464614868,
+      "rewards/accuracies": 0.951171875,
+      "rewards/chosen": -0.023895932361483574,
+      "rewards/margins": 0.2993316650390625,
+      "rewards/rejected": -0.3232276141643524,
+      "sft_loss": 0.1525370478630066,
+      "step": 10
+    },
+    {
+      "epoch": 0.04315841098577734,
+      "grad_norm": 1.7890625,
+      "learning_rate": 1.2692307692307693e-05,
+      "logits/chosen": -2.973320960998535,
+      "logits/rejected": -2.973318576812744,
+      "logps/chosen": -0.2526458501815796,
+      "logps/rejected": -3.23311448097229,
+      "loss": 0.2657,
+      "odds_ratio_loss": 1.0119987726211548,
+      "rewards/accuracies": 0.9423828125,
+      "rewards/chosen": -0.02526458539068699,
+      "rewards/margins": 0.29804685711860657,
+      "rewards/rejected": -0.323311448097229,
+      "sft_loss": 0.16451705992221832,
+      "step": 11
+    },
+    {
+      "epoch": 0.047081902893575285,
+      "grad_norm": 0.95703125,
+      "learning_rate": 1.3846153846153847e-05,
+      "logits/chosen": -2.9654839038848877,
+      "logits/rejected": -2.965482711791992,
+      "logps/chosen": -0.23671554028987885,
+      "logps/rejected": -3.1521968841552734,
+      "loss": 0.2482,
+      "odds_ratio_loss": 0.9054768085479736,
+      "rewards/accuracies": 0.9580078125,
+      "rewards/chosen": -0.023671552538871765,
+      "rewards/margins": 0.2915481626987457,
+      "rewards/rejected": -0.3152197301387787,
+      "sft_loss": 0.15767298638820648,
+      "step": 12
+    },
+    {
+      "epoch": 0.05100539480137322,
+      "grad_norm": 1.078125,
+      "learning_rate": 1.5e-05,
+      "logits/chosen": -2.965122699737549,
+      "logits/rejected": -2.965108633041382,
+      "logps/chosen": -0.27447113394737244,
+      "logps/rejected": -3.1599385738372803,
+      "loss": 0.2891,
+      "odds_ratio_loss": 0.9649019241333008,
+      "rewards/accuracies": 0.9453125,
+      "rewards/chosen": -0.027447111904621124,
+      "rewards/margins": 0.28854674100875854,
+      "rewards/rejected": -0.31599387526512146,
+      "sft_loss": 0.19265753030776978,
+      "step": 13
+    },
+    {
+      "epoch": 0.05492888670917116,
+      "grad_norm": 0.93359375,
+      "learning_rate": 1.6153846153846154e-05,
+      "logits/chosen": -2.9600794315338135,
+      "logits/rejected": -2.9600796699523926,
+      "logps/chosen": -0.248041033744812,
+      "logps/rejected": -3.1299993991851807,
+      "loss": 0.2603,
+      "odds_ratio_loss": 0.9177720546722412,
+      "rewards/accuracies": 0.9580078125,
+      "rewards/chosen": -0.02480410598218441,
+      "rewards/margins": 0.28819578886032104,
+      "rewards/rejected": -0.3129999339580536,
+      "sft_loss": 0.16850169003009796,
+      "step": 14
     },
     {
       "epoch": 0.058852378616969105,
-      "grad_norm": 1.203125,
-      "learning_rate": 0.0001764705882352941,
-      "logits/chosen": -3.330181837081909,
-      "logits/rejected": -3.3302066326141357,
-      "logps/chosen": -0.2974868416786194,
-      "logps/rejected": -2.9257004261016846,
-      "loss": 0.3131,
-      "odds_ratio_loss": 1.1069270372390747,
-      "rewards/accuracies": 0.9404296875,
-      "rewards/chosen": -0.029748687520623207,
-      "rewards/margins": 0.26282134652137756,
-      "rewards/rejected": -0.292570024728775,
-      "sft_loss": 0.2024519443511963,
-      "step": 30
+      "grad_norm": 0.76171875,
+      "learning_rate": 1.7307692307692306e-05,
+      "logits/chosen": -2.9557976722717285,
+      "logits/rejected": -2.9557933807373047,
+      "logps/chosen": -0.20853769779205322,
+      "logps/rejected": -3.0895190238952637,
+      "loss": 0.2192,
+      "odds_ratio_loss": 0.8289995789527893,
+      "rewards/accuracies": 0.9609375,
+      "rewards/chosen": -0.020853767171502113,
+      "rewards/margins": 0.28809815645217896,
+      "rewards/rejected": -0.30895188450813293,
+      "sft_loss": 0.13633377850055695,
+      "step": 15
+    },
+    {
+      "epoch": 0.06277587052476705,
+      "grad_norm": 0.890625,
+      "learning_rate": 1.8461538461538465e-05,
+      "logits/chosen": -2.95212984085083,
+      "logits/rejected": -2.952127456665039,
+      "logps/chosen": -0.24015557765960693,
+      "logps/rejected": -3.1777472496032715,
+      "loss": 0.253,
+      "odds_ratio_loss": 0.8741129040718079,
+      "rewards/accuracies": 0.9541015625,
+      "rewards/chosen": -0.024015557020902634,
+      "rewards/margins": 0.29375916719436646,
+      "rewards/rejected": -0.3177747428417206,
+      "sft_loss": 0.16556787490844727,
+      "step": 16
+    },
+    {
+      "epoch": 0.06669936243256498,
+      "grad_norm": 0.66015625,
+      "learning_rate": 1.9615384615384617e-05,
+      "logits/chosen": -2.9487290382385254,
+      "logits/rejected": -2.948725461959839,
+      "logps/chosen": -0.22024491429328918,
+      "logps/rejected": -3.160766124725342,
+      "loss": 0.2323,
+      "odds_ratio_loss": 0.8908261656761169,
+      "rewards/accuracies": 0.958984375,
+      "rewards/chosen": -0.02202449180185795,
+      "rewards/margins": 0.2940521240234375,
+      "rewards/rejected": -0.3160766065120697,
+      "sft_loss": 0.14318522810935974,
+      "step": 17
+    },
+    {
+      "epoch": 0.07062285434036292,
+      "grad_norm": 0.59375,
+      "learning_rate": 2.076923076923077e-05,
+      "logits/chosen": -2.9467873573303223,
+      "logits/rejected": -2.9467759132385254,
+      "logps/chosen": -0.1970514953136444,
+      "logps/rejected": -3.207763195037842,
+      "loss": 0.2066,
+      "odds_ratio_loss": 0.8174700140953064,
+      "rewards/accuracies": 0.96484375,
+      "rewards/chosen": -0.0197051502764225,
+      "rewards/margins": 0.3010711669921875,
+      "rewards/rejected": -0.3207763433456421,
+      "sft_loss": 0.12490144371986389,
+      "step": 18
+    },
+    {
+      "epoch": 0.07454634624816087,
+      "grad_norm": 1.0,
+      "learning_rate": 2.1923076923076924e-05,
+      "logits/chosen": -2.9435489177703857,
+      "logits/rejected": -2.9435434341430664,
+      "logps/chosen": -0.2126975655555725,
+      "logps/rejected": -3.2167563438415527,
+      "loss": 0.2224,
+      "odds_ratio_loss": 0.8775886297225952,
+      "rewards/accuracies": 0.966796875,
+      "rewards/chosen": -0.02126975730061531,
+      "rewards/margins": 0.30040591955184937,
+      "rewards/rejected": -0.3216756284236908,
+      "sft_loss": 0.13463406264781952,
+      "step": 19
     },
     {
       "epoch": 0.0784698381559588,
-      "grad_norm": 2.25,
-      "learning_rate": 0.0002352941176470588,
-      "logits/chosen": -3.2383294105529785,
-      "logits/rejected": -3.238351345062256,
-      "logps/chosen": -0.2954941689968109,
-      "logps/rejected": -3.020512580871582,
-      "loss": 0.311,
-      "odds_ratio_loss": 1.1244500875473022,
-      "rewards/accuracies": 0.9439452886581421,
-      "rewards/chosen": -0.02954941987991333,
-      "rewards/margins": 0.2725018560886383,
-      "rewards/rejected": -0.30205124616622925,
-      "sft_loss": 0.19855467975139618,
-      "step": 40
+      "grad_norm": 0.8203125,
+      "learning_rate": 2.307692307692308e-05,
+      "logits/chosen": -2.939420223236084,
+      "logits/rejected": -2.939415216445923,
+      "logps/chosen": -0.21315571665763855,
+      "logps/rejected": -3.131246328353882,
+      "loss": 0.2229,
+      "odds_ratio_loss": 0.8950154781341553,
+      "rewards/accuracies": 0.9619140625,
+      "rewards/chosen": -0.021315572783350945,
+      "rewards/margins": 0.29180908203125,
+      "rewards/rejected": -0.3131246566772461,
+      "sft_loss": 0.13337606191635132,
+      "step": 20
+    },
+    {
+      "epoch": 0.08239333006375674,
+      "grad_norm": 0.69140625,
+      "learning_rate": 2.423076923076923e-05,
+      "logits/chosen": -2.9376261234283447,
+      "logits/rejected": -2.9376211166381836,
+      "logps/chosen": -0.2413405179977417,
+      "logps/rejected": -3.1577987670898438,
+      "loss": 0.2519,
+      "odds_ratio_loss": 0.9613508582115173,
+      "rewards/accuracies": 0.94921875,
+      "rewards/chosen": -0.02413405105471611,
+      "rewards/margins": 0.2916458249092102,
+      "rewards/rejected": -0.3157798647880554,
+      "sft_loss": 0.15580427646636963,
+      "step": 21
+    },
+    {
+      "epoch": 0.08631682197155469,
+      "grad_norm": 1.125,
+      "learning_rate": 2.5384615384615386e-05,
+      "logits/chosen": -2.9448554515838623,
+      "logits/rejected": -2.944854974746704,
+      "logps/chosen": -0.2407849133014679,
+      "logps/rejected": -3.2201244831085205,
+      "loss": 0.252,
+      "odds_ratio_loss": 0.9313341379165649,
+      "rewards/accuracies": 0.962890625,
+      "rewards/chosen": -0.02407849207520485,
+      "rewards/margins": 0.297933965921402,
+      "rewards/rejected": -0.32201245427131653,
+      "sft_loss": 0.15885500609874725,
+      "step": 22
+    },
+    {
+      "epoch": 0.09024031387935262,
+      "grad_norm": 1.4375,
+      "learning_rate": 2.6538461538461538e-05,
+      "logits/chosen": -2.946904182434082,
+      "logits/rejected": -2.946908473968506,
+      "logps/chosen": -0.2284041941165924,
+      "logps/rejected": -3.2244067192077637,
+      "loss": 0.242,
+      "odds_ratio_loss": 1.0077147483825684,
+      "rewards/accuracies": 0.947265625,
+      "rewards/chosen": -0.02284042164683342,
+      "rewards/margins": 0.2996002435684204,
+      "rewards/rejected": -0.32244062423706055,
+      "sft_loss": 0.14126919209957123,
+      "step": 23
+    },
+    {
+      "epoch": 0.09416380578715057,
+      "grad_norm": 0.8046875,
+      "learning_rate": 2.7692307692307694e-05,
+      "logits/chosen": -2.9473867416381836,
+      "logits/rejected": -2.947389602661133,
+      "logps/chosen": -0.22177951037883759,
+      "logps/rejected": -3.1969687938690186,
+      "loss": 0.2362,
+      "odds_ratio_loss": 0.8804333209991455,
+      "rewards/accuracies": 0.94140625,
+      "rewards/chosen": -0.02217794954776764,
+      "rewards/margins": 0.29751890897750854,
+      "rewards/rejected": -0.31969690322875977,
+      "sft_loss": 0.14815130829811096,
+      "step": 24
     },
     {
       "epoch": 0.0980872976949485,
-      "grad_norm": 1.1953125,
-      "learning_rate": 0.0002941176470588235,
-      "logits/chosen": -3.1789066791534424,
-      "logits/rejected": -3.1789205074310303,
-      "logps/chosen": -0.33446550369262695,
-      "logps/rejected": -2.795537233352661,
-      "loss": 0.3539,
-      "odds_ratio_loss": 1.2969881296157837,
-      "rewards/accuracies": 0.9242187738418579,
-      "rewards/chosen": -0.033446550369262695,
-      "rewards/margins": 0.24610717594623566,
-      "rewards/rejected": -0.27955371141433716,
-      "sft_loss": 0.22420397400856018,
-      "step": 50
+      "grad_norm": 0.9921875,
+      "learning_rate": 2.884615384615385e-05,
+      "logits/chosen": -2.955061197280884,
+      "logits/rejected": -2.9550623893737793,
+      "logps/chosen": -0.22592617571353912,
+      "logps/rejected": -3.314274549484253,
+      "loss": 0.2371,
+      "odds_ratio_loss": 0.9324527978897095,
+      "rewards/accuracies": 0.962890625,
+      "rewards/chosen": -0.022592617198824883,
+      "rewards/margins": 0.30883485078811646,
+      "rewards/rejected": -0.3314274549484253,
+      "sft_loss": 0.14389845728874207,
+      "step": 25
+    },
+    {
+      "epoch": 0.10201078960274644,
+      "grad_norm": 0.75,
+      "learning_rate": 3e-05,
+      "logits/chosen": -2.958292245864868,
+      "logits/rejected": -2.958291530609131,
+      "logps/chosen": -0.2314324676990509,
+      "logps/rejected": -3.2401301860809326,
+      "loss": 0.2422,
+      "odds_ratio_loss": 0.9014456272125244,
+      "rewards/accuracies": 0.955078125,
+      "rewards/chosen": -0.02314324676990509,
+      "rewards/margins": 0.30086976289749146,
+      "rewards/rejected": -0.32401302456855774,
+      "sft_loss": 0.15201067924499512,
+      "step": 26
+    },
+    {
+      "epoch": 0.10593428151054439,
+      "grad_norm": 0.97265625,
+      "learning_rate": 2.9998576083758987e-05,
+      "logits/chosen": -2.970607280731201,
+      "logits/rejected": -2.970606803894043,
+      "logps/chosen": -0.23627859354019165,
+      "logps/rejected": -3.213726043701172,
+      "loss": 0.2479,
+      "odds_ratio_loss": 0.8589389324188232,
+      "rewards/accuracies": 0.955078125,
+      "rewards/chosen": -0.023627860471606255,
+      "rewards/margins": 0.2977447509765625,
+      "rewards/rejected": -0.3213726282119751,
+      "sft_loss": 0.16202089190483093,
+      "step": 27
+    },
+    {
+      "epoch": 0.10985777341834232,
+      "grad_norm": 1.046875,
+      "learning_rate": 2.999430460537427e-05,
+      "logits/chosen": -2.9736104011535645,
+      "logits/rejected": -2.9736101627349854,
+      "logps/chosen": -0.25192198157310486,
+      "logps/rejected": -3.2232658863067627,
+      "loss": 0.2654,
+      "odds_ratio_loss": 0.9881945848464966,
+      "rewards/accuracies": 0.9521484375,
+      "rewards/chosen": -0.025192195549607277,
+      "rewards/margins": 0.2971344292163849,
+      "rewards/rejected": -0.3223266005516052,
+      "sft_loss": 0.16653719544410706,
+      "step": 28
+    },
+    {
+      "epoch": 0.11378126532614026,
+      "grad_norm": 0.85546875,
+      "learning_rate": 2.9987186375809513e-05,
+      "logits/chosen": -2.979647159576416,
+      "logits/rejected": -2.9796371459960938,
+      "logps/chosen": -0.20289547741413116,
+      "logps/rejected": -3.250380754470825,
+      "loss": 0.2129,
+      "odds_ratio_loss": 0.7935500144958496,
+      "rewards/accuracies": 0.96484375,
+      "rewards/chosen": -0.020289547741413116,
+      "rewards/margins": 0.3047485649585724,
+      "rewards/rejected": -0.32503804564476013,
+      "sft_loss": 0.13351328670978546,
+      "step": 29
     },
     {
       "epoch": 0.11770475723393821,
-      "grad_norm": 1.53125,
-      "learning_rate": 0.0002997142559882001,
-      "logits/chosen": -3.172243118286133,
-      "logits/rejected": -3.172255754470825,
-      "logps/chosen": -0.3375310003757477,
-      "logps/rejected": -2.670206308364868,
-      "loss": 0.3562,
-      "odds_ratio_loss": 1.2899665832519531,
-      "rewards/accuracies": 0.9263671636581421,
-      "rewards/chosen": -0.03375309333205223,
-      "rewards/margins": 0.23326751589775085,
-      "rewards/rejected": -0.26702064275741577,
-      "sft_loss": 0.2272227704524994,
-      "step": 60
+      "grad_norm": 0.7265625,
+      "learning_rate": 2.997722274649974e-05,
+      "logits/chosen": -2.9907119274139404,
+      "logits/rejected": -2.990703582763672,
+      "logps/chosen": -0.22006310522556305,
+      "logps/rejected": -3.3016364574432373,
+      "loss": 0.2316,
+      "odds_ratio_loss": 0.9098002910614014,
+      "rewards/accuracies": 0.9619140625,
+      "rewards/chosen": -0.022006310522556305,
+      "rewards/margins": 0.308157354593277,
+      "rewards/rejected": -0.3301636874675751,
+      "sft_loss": 0.14064472913742065,
+      "step": 30
+    },
+    {
+      "epoch": 0.12162824914173614,
+      "grad_norm": 0.703125,
+      "learning_rate": 2.9964415609094767e-05,
+      "logits/chosen": -2.999443769454956,
+      "logits/rejected": -2.9994404315948486,
+      "logps/chosen": -0.21595218777656555,
+      "logps/rejected": -3.3530983924865723,
+      "loss": 0.2264,
+      "odds_ratio_loss": 0.8591948747634888,
+      "rewards/accuracies": 0.9619140625,
+      "rewards/chosen": -0.021595221012830734,
+      "rewards/margins": 0.3137146234512329,
+      "rewards/rejected": -0.33530986309051514,
+      "sft_loss": 0.14047320187091827,
+      "step": 31
+    },
+    {
+      "epoch": 0.1255517410495341,
+      "grad_norm": 0.8359375,
+      "learning_rate": 2.994876739510005e-05,
+      "logits/chosen": -3.005934953689575,
+      "logits/rejected": -3.005929470062256,
+      "logps/chosen": -0.23634353280067444,
+      "logps/rejected": -3.2525179386138916,
+      "loss": 0.2487,
+      "odds_ratio_loss": 0.8775915503501892,
+      "rewards/accuracies": 0.955078125,
+      "rewards/chosen": -0.023634355515241623,
+      "rewards/margins": 0.30161744356155396,
+      "rewards/rejected": -0.3252517879009247,
+      "sft_loss": 0.16090597212314606,
+      "step": 32
+    },
+    {
+      "epoch": 0.12947523295733201,
+      "grad_norm": 0.734375,
+      "learning_rate": 2.993028107541506e-05,
+      "logits/chosen": -3.0072615146636963,
+      "logits/rejected": -3.0072617530822754,
+      "logps/chosen": -0.20827826857566833,
+      "logps/rejected": -3.354121685028076,
+      "loss": 0.2175,
+      "odds_ratio_loss": 0.7500962018966675,
+      "rewards/accuracies": 0.9658203125,
+      "rewards/chosen": -0.020827826112508774,
+      "rewards/margins": 0.314584344625473,
+      "rewards/rejected": -0.3354122042655945,
+      "sft_loss": 0.1424492597579956,
+      "step": 33
+    },
+    {
+      "epoch": 0.13339872486512996,
+      "grad_norm": 0.6875,
+      "learning_rate": 2.9908960159769243e-05,
+      "logits/chosen": -3.0059030055999756,
+      "logits/rejected": -3.0059022903442383,
+      "logps/chosen": -0.21530942618846893,
+      "logps/rejected": -3.425773859024048,
+      "loss": 0.2248,
+      "odds_ratio_loss": 0.8348552584648132,
+      "rewards/accuracies": 0.9599609375,
+      "rewards/chosen": -0.021530942991375923,
+      "rewards/margins": 0.3210464417934418,
+      "rewards/rejected": -0.34257739782333374,
+      "sft_loss": 0.14131930470466614,
+      "step": 34
     },
     {
       "epoch": 0.1373222167729279,
-      "grad_norm": 2.34375,
-      "learning_rate": 0.0002987278965067285,
-      "logits/chosen": -3.221324920654297,
-      "logits/rejected": -3.2213356494903564,
-      "logps/chosen": -0.3481382429599762,
-      "logps/rejected": -2.922685146331787,
-      "loss": 0.3657,
-      "odds_ratio_loss": 1.2384110689163208,
-      "rewards/accuracies": 0.9341796636581421,
-      "rewards/chosen": -0.03481382131576538,
-      "rewards/margins": 0.25745469331741333,
-      "rewards/rejected": -0.2922685146331787,
-      "sft_loss": 0.2418569028377533,
-      "step": 70
+      "grad_norm": 0.5859375,
+      "learning_rate": 2.9884808696055675e-05,
+      "logits/chosen": -3.0060172080993652,
+      "logits/rejected": -3.0060067176818848,
+      "logps/chosen": -0.20461970567703247,
+      "logps/rejected": -3.3735952377319336,
+      "loss": 0.2132,
+      "odds_ratio_loss": 0.7768919467926025,
+      "rewards/accuracies": 0.96484375,
+      "rewards/chosen": -0.020461970940232277,
+      "rewards/margins": 0.31689757108688354,
+      "rewards/rejected": -0.33735954761505127,
+      "sft_loss": 0.13550838828086853,
+      "step": 35
+    },
+    {
+      "epoch": 0.14124570868072583,
+      "grad_norm": 1.0234375,
+      "learning_rate": 2.985783126956255e-05,
+      "logits/chosen": -3.008485794067383,
+      "logits/rejected": -3.008484125137329,
+      "logps/chosen": -0.22423098981380463,
+      "logps/rejected": -3.372241258621216,
+      "loss": 0.2359,
+      "odds_ratio_loss": 0.8862612247467041,
+      "rewards/accuracies": 0.9541015625,
+      "rewards/chosen": -0.022423099726438522,
+      "rewards/margins": 0.31480103731155396,
+      "rewards/rejected": -0.3372241258621216,
+      "sft_loss": 0.1473112255334854,
+      "step": 36
+    },
+    {
+      "epoch": 0.14516920058852378,
+      "grad_norm": 0.70703125,
+      "learning_rate": 2.9828033002102624e-05,
+      "logits/chosen": -3.004643201828003,
+      "logits/rejected": -3.004650831222534,
+      "logps/chosen": -0.23787802457809448,
+      "logps/rejected": -3.5523312091827393,
+      "loss": 0.249,
+      "odds_ratio_loss": 0.8607804775238037,
+      "rewards/accuracies": 0.96484375,
+      "rewards/chosen": -0.02378780208528042,
+      "rewards/margins": 0.3314453065395355,
+      "rewards/rejected": -0.3552331328392029,
+      "sft_loss": 0.16296614706516266,
+      "step": 37
+    },
+    {
+      "epoch": 0.14909269249632173,
+      "grad_norm": 2.671875,
+      "learning_rate": 2.9795419551040836e-05,
+      "logits/chosen": -3.001986026763916,
+      "logits/rejected": -3.001978635787964,
+      "logps/chosen": -0.2463943511247635,
+      "logps/rejected": -3.445491075515747,
+      "loss": 0.2589,
+      "odds_ratio_loss": 0.8702651262283325,
+      "rewards/accuracies": 0.947265625,
+      "rewards/chosen": -0.024639436975121498,
+      "rewards/margins": 0.3199096620082855,
+      "rewards/rejected": -0.34454911947250366,
+      "sft_loss": 0.17186668515205383,
+      "step": 38
+    },
+    {
+      "epoch": 0.15301618440411965,
+      "grad_norm": 0.71484375,
+      "learning_rate": 2.9759997108220197e-05,
+      "logits/chosen": -3.000178813934326,
+      "logits/rejected": -3.0001754760742188,
+      "logps/chosen": -0.2103407084941864,
+      "logps/rejected": -3.4846935272216797,
+      "loss": 0.2238,
+      "odds_ratio_loss": 0.8533927202224731,
+      "rewards/accuracies": 0.943359375,
+      "rewards/chosen": -0.02103407122194767,
+      "rewards/margins": 0.32743528485298157,
+      "rewards/rejected": -0.3484693765640259,
+      "sft_loss": 0.13847582042217255,
+      "step": 39
     },
     {
       "epoch": 0.1569396763119176,
-      "grad_norm": 2.578125,
-      "learning_rate": 0.00029704203184486135,
-      "logits/chosen": -3.1355342864990234,
-      "logits/rejected": -3.1355600357055664,
-      "logps/chosen": -0.37931394577026367,
-      "logps/rejected": -2.745541572570801,
-      "loss": 0.398,
-      "odds_ratio_loss": 1.3987653255462646,
-      "rewards/accuracies": 0.926953136920929,
-      "rewards/chosen": -0.037931397557258606,
-      "rewards/margins": 0.23662278056144714,
-      "rewards/rejected": -0.27455419301986694,
-      "sft_loss": 0.2580908536911011,
-      "step": 80
+      "grad_norm": 0.8515625,
+      "learning_rate": 2.972177239878627e-05,
+      "logits/chosen": -2.9975767135620117,
+      "logits/rejected": -2.997580051422119,
+      "logps/chosen": -0.18677090108394623,
+      "logps/rejected": -3.4546234607696533,
+      "loss": 0.1964,
+      "odds_ratio_loss": 0.719777524471283,
+      "rewards/accuracies": 0.9580078125,
+      "rewards/chosen": -0.018677091225981712,
+      "rewards/margins": 0.32678529620170593,
+      "rewards/rejected": -0.3454623818397522,
+      "sft_loss": 0.12438130378723145,
+      "step": 40
+    },
+    {
+      "epoch": 0.16086316821971555,
+      "grad_norm": 0.734375,
+      "learning_rate": 2.968075267991032e-05,
+      "logits/chosen": -2.9942257404327393,
+      "logits/rejected": -2.9942216873168945,
+      "logps/chosen": -0.19570019841194153,
+      "logps/rejected": -3.47206711769104,
+      "loss": 0.2066,
+      "odds_ratio_loss": 0.8000987768173218,
+      "rewards/accuracies": 0.95703125,
+      "rewards/chosen": -0.019570019096136093,
+      "rewards/margins": 0.3276367783546448,
+      "rewards/rejected": -0.3472067415714264,
+      "sft_loss": 0.12656894326210022,
+      "step": 41
+    },
+    {
+      "epoch": 0.16478666012751347,
+      "grad_norm": 0.578125,
+      "learning_rate": 2.9636945739411533e-05,
+      "logits/chosen": -2.9919114112854004,
+      "logits/rejected": -2.9919145107269287,
+      "logps/chosen": -0.19992399215698242,
+      "logps/rejected": -3.5652499198913574,
+      "loss": 0.2114,
+      "odds_ratio_loss": 0.8476812243461609,
+      "rewards/accuracies": 0.9580078125,
+      "rewards/chosen": -0.019992398098111153,
+      "rewards/margins": 0.3365326225757599,
+      "rewards/rejected": -0.3565250039100647,
+      "sft_loss": 0.12663182616233826,
+      "step": 42
+    },
+    {
+      "epoch": 0.16871015203531142,
+      "grad_norm": 0.84375,
+      "learning_rate": 2.9590359894278458e-05,
+      "logits/chosen": -2.989424705505371,
+      "logits/rejected": -2.989422559738159,
+      "logps/chosen": -0.18934307992458344,
+      "logps/rejected": -3.50559663772583,
+      "loss": 0.1983,
+      "odds_ratio_loss": 0.784630298614502,
+      "rewards/accuracies": 0.95703125,
+      "rewards/chosen": -0.018934309482574463,
+      "rewards/margins": 0.331625372171402,
+      "rewards/rejected": -0.35055968165397644,
+      "sft_loss": 0.11984920501708984,
+      "step": 43
+    },
+    {
+      "epoch": 0.17263364394310937,
+      "grad_norm": 0.96875,
+      "learning_rate": 2.9541003989089956e-05,
+      "logits/chosen": -2.990034818649292,
+      "logits/rejected": -2.990030288696289,
+      "logps/chosen": -0.1884869933128357,
+      "logps/rejected": -3.5475568771362305,
+      "loss": 0.1982,
+      "odds_ratio_loss": 0.7193607687950134,
+      "rewards/accuracies": 0.9609375,
+      "rewards/chosen": -0.01884870044887066,
+      "rewards/margins": 0.335906982421875,
+      "rewards/rejected": -0.3547556400299072,
+      "sft_loss": 0.1262793391942978,
+      "step": 44
     },
     {
       "epoch": 0.17655713585090732,
-      "grad_norm": 2.5625,
-      "learning_rate": 0.0002946645910535173,
-      "logits/chosen": -2.493868112564087,
-      "logits/rejected": -2.493875026702881,
-      "logps/chosen": -0.3405473828315735,
-      "logps/rejected": -2.824284553527832,
-      "loss": 0.358,
-      "odds_ratio_loss": 1.2909947633743286,
-      "rewards/accuracies": 0.929492175579071,
-      "rewards/chosen": -0.03405473753809929,
-      "rewards/margins": 0.24837371706962585,
-      "rewards/rejected": -0.28242844343185425,
-      "sft_loss": 0.22891943156719208,
-      "step": 90
+      "grad_norm": 0.57421875,
+      "learning_rate": 2.9488887394336025e-05,
+      "logits/chosen": -2.995880126953125,
+      "logits/rejected": -2.9958770275115967,
+      "logps/chosen": -0.19902820885181427,
+      "logps/rejected": -3.553673028945923,
+      "loss": 0.2075,
+      "odds_ratio_loss": 0.8336374163627625,
+      "rewards/accuracies": 0.9580078125,
+      "rewards/chosen": -0.019902819767594337,
+      "rewards/margins": 0.3354644775390625,
+      "rewards/rejected": -0.3553672730922699,
+      "sft_loss": 0.12417693436145782,
+      "step": 45
+    },
+    {
+      "epoch": 0.18048062775870524,
+      "grad_norm": 0.74609375,
+      "learning_rate": 2.9434020004638757e-05,
+      "logits/chosen": -2.991523265838623,
+      "logits/rejected": -2.991513252258301,
+      "logps/chosen": -0.19020922482013702,
+      "logps/rejected": -3.644005537033081,
+      "loss": 0.2015,
+      "odds_ratio_loss": 0.7555862069129944,
+      "rewards/accuracies": 0.9638671875,
+      "rewards/chosen": -0.019020922482013702,
+      "rewards/margins": 0.34537965059280396,
+      "rewards/rejected": -0.36440056562423706,
+      "sft_loss": 0.12592323124408722,
+      "step": 46
+    },
+    {
+      "epoch": 0.1844041196665032,
+      "grad_norm": 0.96484375,
+      "learning_rate": 2.9376412236873792e-05,
+      "logits/chosen": -2.986288547515869,
+      "logits/rejected": -2.9862923622131348,
+      "logps/chosen": -0.18654456734657288,
+      "logps/rejected": -3.63527512550354,
+      "loss": 0.1949,
+      "odds_ratio_loss": 0.8158512711524963,
+      "rewards/accuracies": 0.9658203125,
+      "rewards/chosen": -0.018654460087418556,
+      "rewards/margins": 0.3448730409145355,
+      "rewards/rejected": -0.3635275065898895,
+      "sft_loss": 0.1132911741733551,
+      "step": 47
+    },
+    {
+      "epoch": 0.18832761157430114,
+      "grad_norm": 0.796875,
+      "learning_rate": 2.931607502819261e-05,
+      "logits/chosen": -2.997182846069336,
+      "logits/rejected": -2.9971840381622314,
+      "logps/chosen": -0.20562656223773956,
+      "logps/rejected": -3.5754082202911377,
+      "loss": 0.2192,
+      "odds_ratio_loss": 0.8831424713134766,
+      "rewards/accuracies": 0.9541015625,
+      "rewards/chosen": -0.020562658086419106,
+      "rewards/margins": 0.33697813749313354,
+      "rewards/rejected": -0.3575408458709717,
+      "sft_loss": 0.1309242844581604,
+      "step": 48
+    },
+    {
+      "epoch": 0.19225110348209906,
+      "grad_norm": 0.8046875,
+      "learning_rate": 2.925301983394607e-05,
+      "logits/chosen": -2.9951722621917725,
+      "logits/rejected": -2.9951741695404053,
+      "logps/chosen": -0.19311097264289856,
+      "logps/rejected": -3.7165184020996094,
+      "loss": 0.2027,
+      "odds_ratio_loss": 0.7433779239654541,
+      "rewards/accuracies": 0.9638671875,
+      "rewards/chosen": -0.019311096519231796,
+      "rewards/margins": 0.3523406982421875,
+      "rewards/rejected": -0.371651828289032,
+      "sft_loss": 0.12833088636398315,
+      "step": 49
     },
     {
       "epoch": 0.196174595389897,
-      "grad_norm": 3.421875,
-      "learning_rate": 0.0002916067558423063,
-      "logits/chosen": -2.403263568878174,
-      "logits/rejected": -2.403261423110962,
-      "logps/chosen": -0.35239094495773315,
-      "logps/rejected": -2.8051955699920654,
-      "loss": 0.3722,
-      "odds_ratio_loss": 1.3476725816726685,
-      "rewards/accuracies": 0.9253906011581421,
-      "rewards/chosen": -0.03523910045623779,
-      "rewards/margins": 0.245280459523201,
-      "rewards/rejected": -0.2805195450782776,
-      "sft_loss": 0.23746605217456818,
-      "step": 100
-    },
-    {
-      "epoch": 0.2157920549288867,
-      "grad_norm": 4.59375,
-      "learning_rate": 0.000287882907989101,
-      "logits/chosen": -2.881706714630127,
-      "logits/rejected": -2.8817214965820312,
-      "logps/chosen": -0.3732207417488098,
-      "logps/rejected": -2.704741954803467,
-      "loss": 0.3954,
-      "odds_ratio_loss": 1.417145013809204,
-      "rewards/accuracies": 0.9146484136581421,
-      "rewards/chosen": -0.03732207417488098,
-      "rewards/margins": 0.2331521213054657,
-      "rewards/rejected": -0.2704741954803467,
-      "sft_loss": 0.2537223696708679,
-      "step": 110
-    },
-    {
-      "epoch": 0.23540951446787642,
-      "grad_norm": 2.15625,
-      "learning_rate": 0.0002835105616988724,
-      "logits/chosen": -2.9882686138153076,
-      "logits/rejected": -2.9882702827453613,
-      "logps/chosen": -0.3399866223335266,
-      "logps/rejected": -2.6247470378875732,
-      "loss": 0.3602,
-      "odds_ratio_loss": 1.3240976333618164,
-      "rewards/accuracies": 0.9248046875,
-      "rewards/chosen": -0.03399866446852684,
-      "rewards/margins": 0.22847604751586914,
-      "rewards/rejected": -0.2624747157096863,
-      "sft_loss": 0.2277795970439911,
-      "step": 120
-    },
-    {
-      "epoch": 0.25502697400686614,
-      "grad_norm": 1.5078125,
-      "learning_rate": 0.00027851028122992275,
-      "logits/chosen": -2.9560532569885254,
-      "logits/rejected": -2.956058979034424,
-      "logps/chosen": -0.3172145485877991,
-      "logps/rejected": -2.878936290740967,
-      "loss": 0.3352,
-      "odds_ratio_loss": 1.214184045791626,
-      "rewards/accuracies": 0.9339843988418579,
-      "rewards/chosen": -0.03172145411372185,
-      "rewards/margins": 0.25617218017578125,
-      "rewards/rejected": -0.2878936529159546,
-      "sft_loss": 0.21379432082176208,
-      "step": 130
-    },
-    {
-      "epoch": 0.2746444335458558,
-      "grad_norm": 2.78125,
-      "learning_rate": 0.0002729055841749404,
-      "logits/chosen": -2.830913543701172,
-      "logits/rejected": -2.830937623977661,
-      "logps/chosen": -0.31502777338027954,
-      "logps/rejected": -2.772010326385498,
-      "loss": 0.3319,
-      "odds_ratio_loss": 1.2316268682479858,
-      "rewards/accuracies": 0.927539050579071,
-      "rewards/chosen": -0.03150278329849243,
-      "rewards/margins": 0.2456982582807541,
-      "rewards/rejected": -0.2772010266780853,
-      "sft_loss": 0.2087615430355072,
-      "step": 140
-    },
-    {
-      "epoch": 0.2942618930848455,
-      "grad_norm": 1.1796875,
-      "learning_rate": 0.0002667228308517715,
-      "logits/chosen": -2.7827162742614746,
-      "logits/rejected": -2.7827162742614746,
-      "logps/chosen": -0.31607183814048767,
-      "logps/rejected": -2.9119043350219727,
-      "loss": 0.3321,
-      "odds_ratio_loss": 1.1889328956604004,
-      "rewards/accuracies": 0.9408203363418579,
-      "rewards/chosen": -0.031607188284397125,
-      "rewards/margins": 0.2595832943916321,
-      "rewards/rejected": -0.2911904454231262,
-      "sft_loss": 0.2131653130054474,
-      "step": 150
-    },
-    {
-      "epoch": 0.3138793526238352,
-      "grad_norm": 1.46875,
-      "learning_rate": 0.0002599911003241308,
-      "logits/chosen": -2.828200101852417,
-      "logits/rejected": -2.828172206878662,
-      "logps/chosen": -0.3029854893684387,
-      "logps/rejected": -2.940296173095703,
-      "loss": 0.3187,
-      "odds_ratio_loss": 1.1783697605133057,
-      "rewards/accuracies": 0.9388672113418579,
-      "rewards/chosen": -0.030298549681901932,
-      "rewards/margins": 0.26373106241226196,
-      "rewards/rejected": -0.2940296232700348,
-      "sft_loss": 0.20090630650520325,
-      "step": 160
-    },
-    {
-      "epoch": 0.3334968121628249,
-      "grad_norm": 1.6875,
-      "learning_rate": 0.0002527420536353579,
-      "logits/chosen": -2.8486075401306152,
-      "logits/rejected": -2.8485991954803467,
-      "logps/chosen": -0.2984660565853119,
-      "logps/rejected": -2.9544076919555664,
-      "loss": 0.3145,
-      "odds_ratio_loss": 1.185927152633667,
-      "rewards/accuracies": 0.9361327886581421,
-      "rewards/chosen": -0.02984660305082798,
-      "rewards/margins": 0.2655941843986511,
-      "rewards/rejected": -0.29544076323509216,
-      "sft_loss": 0.19592249393463135,
-      "step": 170
-    },
-    {
-      "epoch": 0.35311427170181464,
-      "grad_norm": 2.8125,
-      "learning_rate": 0.00024500978489846596,
-      "logits/chosen": -2.9189867973327637,
-      "logits/rejected": -2.918994188308716,
-      "logps/chosen": -0.3076332211494446,
-      "logps/rejected": -3.066086530685425,
-      "loss": 0.3244,
-      "odds_ratio_loss": 1.1991689205169678,
-      "rewards/accuracies": 0.9306640625,
-      "rewards/chosen": -0.030763322487473488,
-      "rewards/margins": 0.27584534883499146,
-      "rewards/rejected": -0.306608647108078,
-      "sft_loss": 0.20451626181602478,
-      "step": 180
-    },
-    {
-      "epoch": 0.37273173124080433,
-      "grad_norm": 1.3828125,
-      "learning_rate": 0.00023683066094284426,
-      "logits/chosen": -2.911583423614502,
-      "logits/rejected": -2.9115893840789795,
-      "logps/chosen": -0.2833148241043091,
-      "logps/rejected": -3.1591389179229736,
-      "loss": 0.2994,
-      "odds_ratio_loss": 1.1305713653564453,
-      "rewards/accuracies": 0.9390624761581421,
-      "rewards/chosen": -0.02833147905766964,
-      "rewards/margins": 0.2875823974609375,
-      "rewards/rejected": -0.3159138560295105,
-      "sft_loss": 0.186366468667984,
-      "step": 190
-    },
-    {
-      "epoch": 0.392349190779794,
-      "grad_norm": 1.03125,
-      "learning_rate": 0.00022824315027179597,
-      "logits/chosen": -2.9097139835357666,
-      "logits/rejected": -2.90971040725708,
-      "logps/chosen": -0.28494173288345337,
-      "logps/rejected": -2.9840199947357178,
-      "loss": 0.2999,
-      "odds_ratio_loss": 1.0984803438186646,
-      "rewards/accuracies": 0.9439452886581421,
-      "rewards/chosen": -0.028494173660874367,
-      "rewards/margins": 0.2699078619480133,
-      "rewards/rejected": -0.29840201139450073,
-      "sft_loss": 0.19001488387584686,
-      "step": 200
+      "grad_norm": 0.859375,
+      "learning_rate": 2.9187258625509518e-05,
+      "logits/chosen": -2.9987900257110596,
+      "logits/rejected": -2.998793601989746,
+      "logps/chosen": -0.2426375150680542,
+      "logps/rejected": -3.5826642513275146,
+      "loss": 0.2549,
+      "odds_ratio_loss": 0.9904361963272095,
+      "rewards/accuracies": 0.9501953125,
+      "rewards/chosen": -0.02426375262439251,
+      "rewards/margins": 0.33400267362594604,
+      "rewards/rejected": -0.3582664132118225,
+      "sft_loss": 0.1558777540922165,
+      "step": 50
     }
   ],
-  "logging_steps": 10,
-  "max_steps": 509,
+  "logging_steps": 1,
+  "max_steps": 254,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
-  "save_steps": 100,
+  "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -366,7 +876,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.581922179792175e+18,
+  "total_flos": 7.909585740421202e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null