Model save

README.md

@@ -17,15 +17,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [HuggingFaceH4/mistral-7b-sft-beta](https://huggingface.co/HuggingFaceH4/mistral-7b-sft-beta) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0675
-- Rewards/chosen: -2.4788
-- Rewards/rejected: -2.9505
-- Rewards/accuracies: 0.6406
-- Rewards/margins: 0.4717
-- Logps/rejected: -552.4012
-- Logps/chosen: -504.9170
-- Logits/rejected: -2.1295
-- Logits/chosen: -2.1638
+- Loss: 0.4965
+- Rewards/chosen: -2.9708
+- Rewards/rejected: -4.3017
+- Rewards/accuracies: 0.7695
+- Rewards/margins: 1.3309
+- Logps/rejected: -687.5271
+- Logps/chosen: -554.1226
+- Logits/rejected: -0.1928
+- Logits/chosen: -0.6531
 
 ## Model description
 
@@ -47,7 +47,7 @@ The following hyperparameters were used during training:
 - learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
-- seed: 5
+- seed: 3
 - distributed_type: multi-GPU
 - num_devices: 8
 - gradient_accumulation_steps: 2
@@ -62,10 +62,14 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.067         | 0.25  | 100  | 0.1174          | -1.4873        | -1.7314          | 0.6133             | 0.2442          | -430.4969      | -405.7653    | -2.3244         | -2.3408       |
-| 0.0435        | 0.49  | 200  | 0.0799          | -2.1802        | -2.5492          | 0.6211             | 0.3690          | -512.2731      | -475.0585    | -2.1421         | -2.1734       |
-| 0.0288        | 0.74  | 300  | 0.0710          | -2.4383        | -2.9105          | 0.6172             | 0.4722          | -548.4017      | -500.8697    | -2.1339         | -2.1675       |
-| 0.032         | 0.99  | 400  | 0.0675          | -2.4788        | -2.9505          | 0.6406             | 0.4717          | -552.4012      | -504.9170    | -2.1295         | -2.1638       |
+| 0.5326        | 0.11  | 100  | 0.6180          | -0.4024        | -0.6993          | 0.6797             | 0.2969          | -327.2873      | -297.2842    | -2.5800         | -2.5958       |
+| 0.4709        | 0.23  | 200  | 0.5608          | -1.1383        | -1.7616          | 0.7109             | 0.6233          | -433.5121      | -370.8716    | -2.1515         | -2.1720       |
+| 0.4289        | 0.34  | 300  | 0.5293          | -1.5404        | -2.3958          | 0.7539             | 0.8554          | -496.9380      | -411.0811    | -2.0882         | -2.1204       |
+| 0.4195        | 0.45  | 400  | 0.5096          | -1.7916        | -2.8995          | 0.7812             | 1.1079          | -547.3041      | -436.1970    | -1.0571         | -1.2976       |
+| 0.3891        | 0.57  | 500  | 0.5086          | -2.6047        | -3.9255          | 0.7812             | 1.3208          | -649.9016      | -517.5072    | -0.8608         | -1.1314       |
+| 0.4182        | 0.68  | 600  | 0.4976          | -2.4968        | -3.7962          | 0.7695             | 1.2994          | -636.9742      | -506.7195    | -0.4354         | -0.8384       |
+| 0.3845        | 0.79  | 700  | 0.4967          | -2.6976        | -4.0084          | 0.7695             | 1.3108          | -658.1885      | -526.7999    | -0.2826         | -0.7200       |
+| 0.3896        | 0.91  | 800  | 0.4965          | -2.9708        | -4.3017          | 0.7695             | 1.3309          | -687.5271      | -554.1226    | -0.1928         | -0.6531       |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.07294640629379838,
-    "train_runtime": 3765.6331,
-    "train_samples": 51894,
-    "train_samples_per_second": 13.781,
-    "train_steps_per_second": 0.108
+    "train_loss": 0.43856339019935237,
+    "train_runtime": 7937.4578,
+    "train_samples": 113028,
+    "train_samples_per_second": 14.24,
+    "train_steps_per_second": 0.111
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c975a67a6eda1c94e6b5d34eee82ff0a365b06c38ca419207b5a3cbb4d49613b
+oid sha256:5e86dbc79da5b6221b132cbc04faab8b97a554c9ca39df05f8010da50192c1d5
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff29be45fbf4dc2f3ea1a73522b7ec72c85a2b7374393b7e454e465b85543885
+oid sha256:48e30d3b8da1f10b4ad9f2abada379c9f4a92e61d6a7079ac884d33bf9e5d6d5
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf3dea0f30a8c199a5dc983dd433e00255a8a010974837b08cc4d7f1b0d14b89
+oid sha256:331cb75e2dc5f4ae29337f4eff1c5fad0704c34b95053780a0f84d881b68c2a4
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.07294640629379838,
-    "train_runtime": 3765.6331,
-    "train_samples": 51894,
-    "train_samples_per_second": 13.781,
-    "train_steps_per_second": 0.108
+    "train_loss": 0.43856339019935237,
+    "train_runtime": 7937.4578,
+    "train_samples": 113028,
+    "train_samples_per_second": 14.24,
+    "train_steps_per_second": 0.111
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.998766954377312,
+  "epoch": 0.9994340690435767,
   "eval_steps": 100,
-  "global_step": 405,
+  "global_step": 883,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 1.2195121951219512e-08,
-      "logits/chosen": -2.8695335388183594,
-      "logits/rejected": -2.8522377014160156,
-      "logps/chosen": -537.80126953125,
-      "logps/rejected": -108.91968536376953,
-      "loss": 0.3287,
+      "learning_rate": 5.617977528089887e-09,
+      "logits/chosen": -2.604583740234375,
+      "logits/rejected": -2.6370604038238525,
+      "logps/chosen": -392.0871887207031,
+      "logps/rejected": -333.6990966796875,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -23,641 +23,1377 @@
       "step": 1
     },
     {
-      "epoch": 0.02,
-      "learning_rate": 1.219512195121951e-07,
-      "logits/chosen": -2.8006999492645264,
-      "logits/rejected": -2.7513413429260254,
-      "logps/chosen": -339.1315612792969,
-      "logps/rejected": -113.41014862060547,
-      "loss": 0.3429,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0010660986881703138,
-      "rewards/margins": 0.0017727299127727747,
-      "rewards/rejected": -0.0007066310499794781,
+      "epoch": 0.01,
+      "learning_rate": 5.617977528089887e-08,
+      "logits/chosen": -2.798471212387085,
+      "logits/rejected": -2.7608420848846436,
+      "logps/chosen": -291.8177490234375,
+      "logps/rejected": -192.53457641601562,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4583333432674408,
+      "rewards/chosen": -0.00019937421893700957,
+      "rewards/margins": -0.00015055020048748702,
+      "rewards/rejected": -4.882401117356494e-05,
       "step": 10
     },
     {
-      "epoch": 0.05,
-      "learning_rate": 2.439024390243902e-07,
-      "logits/chosen": -2.8162312507629395,
-      "logits/rejected": -2.8079066276550293,
-      "logps/chosen": -435.261962890625,
-      "logps/rejected": -116.0378189086914,
-      "loss": 0.3332,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.021241962909698486,
-      "rewards/margins": 0.038096584379673004,
-      "rewards/rejected": -0.016854625195264816,
+      "epoch": 0.02,
+      "learning_rate": 1.1235955056179774e-07,
+      "logits/chosen": -2.8122811317443848,
+      "logits/rejected": -2.7911267280578613,
+      "logps/chosen": -334.42919921875,
+      "logps/rejected": -200.9188995361328,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.001965261297300458,
+      "rewards/margins": 0.0033602300100028515,
+      "rewards/rejected": -0.001394969061948359,
       "step": 20
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 3.6585365853658536e-07,
-      "logits/chosen": -2.7254586219787598,
-      "logits/rejected": -2.688169002532959,
-      "logps/chosen": -437.81072998046875,
-      "logps/rejected": -141.1324920654297,
-      "loss": 0.309,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.08655615150928497,
-      "rewards/margins": 0.20261511206626892,
-      "rewards/rejected": -0.11605894565582275,
+      "epoch": 0.03,
+      "learning_rate": 1.6853932584269663e-07,
+      "logits/chosen": -2.775857448577881,
+      "logits/rejected": -2.73417329788208,
+      "logps/chosen": -372.3719482421875,
+      "logps/rejected": -164.69215393066406,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.014180044643580914,
+      "rewards/margins": 0.02330140210688114,
+      "rewards/rejected": -0.009121356531977654,
       "step": 30
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 4.878048780487804e-07,
-      "logits/chosen": -2.5956244468688965,
-      "logits/rejected": -2.5760390758514404,
-      "logps/chosen": -414.177490234375,
-      "logps/rejected": -168.2574005126953,
-      "loss": 0.2577,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.02355712652206421,
-      "rewards/margins": 0.46499890089035034,
-      "rewards/rejected": -0.4414418339729309,
+      "epoch": 0.05,
+      "learning_rate": 2.2471910112359549e-07,
+      "logits/chosen": -2.775045871734619,
+      "logits/rejected": -2.7430758476257324,
+      "logps/chosen": -326.87652587890625,
+      "logps/rejected": -221.24282836914062,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.01945655047893524,
+      "rewards/margins": 0.05223391577601433,
+      "rewards/rejected": -0.03277735784649849,
       "step": 40
     },
     {
-      "epoch": 0.12,
-      "learning_rate": 4.992461696250783e-07,
-      "logits/chosen": -2.474365711212158,
-      "logits/rejected": -2.461667537689209,
-      "logps/chosen": -420.9219665527344,
-      "logps/rejected": -210.11111450195312,
-      "loss": 0.1873,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -0.11808023601770401,
-      "rewards/margins": 0.785495400428772,
-      "rewards/rejected": -0.903575599193573,
+      "epoch": 0.06,
+      "learning_rate": 2.8089887640449437e-07,
+      "logits/chosen": -2.652109146118164,
+      "logits/rejected": -2.6346287727355957,
+      "logps/chosen": -323.96392822265625,
+      "logps/rejected": -199.55160522460938,
+      "loss": 0.6411,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.019086791202425957,
+      "rewards/margins": 0.12331440299749374,
+      "rewards/rejected": -0.10422760248184204,
       "step": 50
     },
     {
-      "epoch": 0.15,
-      "learning_rate": 4.966461721767899e-07,
-      "logits/chosen": -2.4058423042297363,
-      "logits/rejected": -2.3743529319763184,
-      "logps/chosen": -377.65484619140625,
-      "logps/rejected": -207.171142578125,
-      "loss": 0.1202,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.32305708527565,
-      "rewards/margins": 0.8489507436752319,
-      "rewards/rejected": -1.1720077991485596,
+      "epoch": 0.07,
+      "learning_rate": 3.3707865168539325e-07,
+      "logits/chosen": -2.6026113033294678,
+      "logits/rejected": -2.593480110168457,
+      "logps/chosen": -313.3958740234375,
+      "logps/rejected": -222.98287963867188,
+      "loss": 0.6195,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.029142415151000023,
+      "rewards/margins": 0.17074736952781677,
+      "rewards/rejected": -0.19988977909088135,
       "step": 60
     },
     {
-      "epoch": 0.17,
-      "learning_rate": 4.922100518015975e-07,
-      "logits/chosen": -2.4014129638671875,
-      "logits/rejected": -2.365219831466675,
-      "logps/chosen": -448.73516845703125,
-      "logps/rejected": -269.4298400878906,
-      "loss": 0.0894,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -0.4889157712459564,
-      "rewards/margins": 1.191009759902954,
-      "rewards/rejected": -1.679925560951233,
+      "epoch": 0.08,
+      "learning_rate": 3.9325842696629214e-07,
+      "logits/chosen": -2.5947117805480957,
+      "logits/rejected": -2.569990634918213,
+      "logps/chosen": -395.15338134765625,
+      "logps/rejected": -245.3045196533203,
+      "loss": 0.5814,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.006087436340749264,
+      "rewards/margins": 0.3716045022010803,
+      "rewards/rejected": -0.37769192457199097,
       "step": 70
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 4.859708325770919e-07,
-      "logits/chosen": -2.3665881156921387,
-      "logits/rejected": -2.313610792160034,
-      "logps/chosen": -442.3289489746094,
-      "logps/rejected": -284.679931640625,
-      "loss": 0.0863,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -0.31383419036865234,
-      "rewards/margins": 1.4407079219818115,
-      "rewards/rejected": -1.7545421123504639,
+      "epoch": 0.09,
+      "learning_rate": 4.4943820224719097e-07,
+      "logits/chosen": -2.5502541065216064,
+      "logits/rejected": -2.5388641357421875,
+      "logps/chosen": -367.24169921875,
+      "logps/rejected": -243.64401245117188,
+      "loss": 0.5683,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.05494023486971855,
+      "rewards/margins": 0.48627549409866333,
+      "rewards/rejected": -0.5412156581878662,
       "step": 80
     },
     {
-      "epoch": 0.22,
-      "learning_rate": 4.779749614980225e-07,
-      "logits/chosen": -2.3862123489379883,
-      "logits/rejected": -2.336027145385742,
-      "logps/chosen": -456.930419921875,
-      "logps/rejected": -332.8990478515625,
-      "loss": 0.0628,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.7161350846290588,
-      "rewards/margins": 1.5251940488815308,
-      "rewards/rejected": -2.2413289546966553,
+      "epoch": 0.1,
+      "learning_rate": 4.999980431020109e-07,
+      "logits/chosen": -2.5524516105651855,
+      "logits/rejected": -2.554391860961914,
+      "logps/chosen": -351.7366943359375,
+      "logps/rejected": -259.1504821777344,
+      "loss": 0.5401,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.16875556111335754,
+      "rewards/margins": 0.5268481373786926,
+      "rewards/rejected": -0.6956037282943726,
       "step": 90
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 4.682819627081427e-07,
-      "logits/chosen": -2.3527557849884033,
-      "logits/rejected": -2.2716236114501953,
-      "logps/chosen": -491.0079650878906,
-      "logps/rejected": -360.1409912109375,
-      "loss": 0.067,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.676110029220581,
-      "rewards/margins": 1.8081252574920654,
-      "rewards/rejected": -2.4842352867126465,
+      "epoch": 0.11,
+      "learning_rate": 4.997632524101301e-07,
+      "logits/chosen": -2.595895767211914,
+      "logits/rejected": -2.5705056190490723,
+      "logps/chosen": -351.92919921875,
+      "logps/rejected": -244.37753295898438,
+      "loss": 0.5326,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.17635223269462585,
+      "rewards/margins": 0.589404821395874,
+      "rewards/rejected": -0.7657570838928223,
       "step": 100
     },
     {
-      "epoch": 0.25,
-      "eval_logits/chosen": -2.340771198272705,
-      "eval_logits/rejected": -2.3243579864501953,
-      "eval_logps/chosen": -405.76531982421875,
-      "eval_logps/rejected": -430.49688720703125,
-      "eval_loss": 0.11736096441745758,
-      "eval_rewards/accuracies": 0.61328125,
-      "eval_rewards/chosen": -1.4872568845748901,
-      "eval_rewards/margins": 0.2441793829202652,
-      "eval_rewards/rejected": -1.7314363718032837,
-      "eval_runtime": 53.3203,
-      "eval_samples_per_second": 37.509,
-      "eval_steps_per_second": 0.6,
+      "epoch": 0.11,
+      "eval_logits/chosen": -2.595787525177002,
+      "eval_logits/rejected": -2.5800230503082275,
+      "eval_logps/chosen": -297.28424072265625,
+      "eval_logps/rejected": -327.2872619628906,
+      "eval_loss": 0.6179993748664856,
+      "eval_rewards/accuracies": 0.6796875,
+      "eval_rewards/chosen": -0.40244585275650024,
+      "eval_rewards/margins": 0.2968939244747162,
+      "eval_rewards/rejected": -0.699339747428894,
+      "eval_runtime": 53.2358,
+      "eval_samples_per_second": 37.569,
+      "eval_steps_per_second": 0.601,
       "step": 100
     },
     {
-      "epoch": 0.27,
-      "learning_rate": 4.569639943810477e-07,
-      "logits/chosen": -2.361056089401245,
-      "logits/rejected": -2.282515287399292,
-      "logps/chosen": -496.8377380371094,
-      "logps/rejected": -344.2028503417969,
-      "loss": 0.0656,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.5422588586807251,
-      "rewards/margins": 1.8025261163711548,
-      "rewards/rejected": -2.344785213470459,
+      "epoch": 0.12,
+      "learning_rate": 4.991375032514749e-07,
+      "logits/chosen": -2.571720600128174,
+      "logits/rejected": -2.509852886199951,
+      "logps/chosen": -345.0278015136719,
+      "logps/rejected": -304.7156677246094,
+      "loss": 0.4987,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.33357003331184387,
+      "rewards/margins": 0.8169819712638855,
+      "rewards/rejected": -1.1505521535873413,
       "step": 110
     },
     {
-      "epoch": 0.3,
-      "learning_rate": 4.4410531154874543e-07,
-      "logits/chosen": -2.31123685836792,
-      "logits/rejected": -2.2176012992858887,
-      "logps/chosen": -541.8471069335938,
-      "logps/rejected": -430.7569885253906,
-      "loss": 0.0355,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.0784400701522827,
-      "rewards/margins": 2.053567409515381,
-      "rewards/rejected": -3.132007360458374,
+      "epoch": 0.14,
+      "learning_rate": 4.98121775121344e-07,
+      "logits/chosen": -2.5141258239746094,
+      "logits/rejected": -2.4955408573150635,
+      "logps/chosen": -368.97735595703125,
+      "logps/rejected": -351.9383239746094,
+      "loss": 0.4716,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.5964471101760864,
+      "rewards/margins": 0.8667058944702148,
+      "rewards/rejected": -1.4631531238555908,
       "step": 120
     },
     {
-      "epoch": 0.32,
-      "learning_rate": 4.298016388768561e-07,
-      "logits/chosen": -2.288428783416748,
-      "logits/rejected": -2.175412178039551,
-      "logps/chosen": -530.9747314453125,
-      "logps/rejected": -429.48785400390625,
-      "loss": 0.0441,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.0411689281463623,
-      "rewards/margins": 1.9912636280059814,
-      "rewards/rejected": -3.0324320793151855,
+      "epoch": 0.15,
+      "learning_rate": 4.96717657955441e-07,
+      "logits/chosen": -2.4679911136627197,
+      "logits/rejected": -2.4308536052703857,
+      "logps/chosen": -375.72186279296875,
+      "logps/rejected": -334.9486999511719,
+      "loss": 0.4703,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.7533982992172241,
+      "rewards/margins": 0.7337401509284973,
+      "rewards/rejected": -1.4871385097503662,
       "step": 130
     },
     {
-      "epoch": 0.35,
-      "learning_rate": 4.1415945805573005e-07,
-      "logits/chosen": -2.3122925758361816,
-      "logits/rejected": -2.227431058883667,
-      "logps/chosen": -509.9908142089844,
-      "logps/rejected": -396.0080871582031,
-      "loss": 0.0515,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -0.6500633358955383,
-      "rewards/margins": 2.154536724090576,
-      "rewards/rejected": -2.804600238800049,
+      "epoch": 0.16,
+      "learning_rate": 4.949273496411216e-07,
+      "logits/chosen": -2.432471513748169,
+      "logits/rejected": -2.3869972229003906,
+      "logps/chosen": -416.4320373535156,
+      "logps/rejected": -364.8056640625,
+      "loss": 0.4724,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.5794404745101929,
+      "rewards/margins": 1.0570305585861206,
+      "rewards/rejected": -1.6364710330963135,
       "step": 140
     },
     {
-      "epoch": 0.37,
-      "learning_rate": 3.972952151123984e-07,
-      "logits/chosen": -2.2672953605651855,
-      "logits/rejected": -2.170297384262085,
-      "logps/chosen": -522.5911254882812,
-      "logps/rejected": -445.1206970214844,
-      "loss": 0.0433,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -0.9818037748336792,
-      "rewards/margins": 2.1947696208953857,
-      "rewards/rejected": -3.1765732765197754,
+      "epoch": 0.17,
+      "learning_rate": 4.927536525770046e-07,
+      "logits/chosen": -2.351346969604492,
+      "logits/rejected": -2.2865800857543945,
+      "logps/chosen": -441.26666259765625,
+      "logps/rejected": -346.3591613769531,
+      "loss": 0.4494,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.8117995262145996,
+      "rewards/margins": 0.9228641390800476,
+      "rewards/rejected": -1.734663724899292,
       "step": 150
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 3.793344535444142e-07,
-      "logits/chosen": -2.3276212215423584,
-      "logits/rejected": -2.2313549518585205,
-      "logps/chosen": -537.3373413085938,
-      "logps/rejected": -405.67559814453125,
-      "loss": 0.0465,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": -0.7096673250198364,
-      "rewards/margins": 2.1882476806640625,
-      "rewards/rejected": -2.8979151248931885,
+      "epoch": 0.18,
+      "learning_rate": 4.901999692863326e-07,
+      "logits/chosen": -2.2583577632904053,
+      "logits/rejected": -2.2541985511779785,
+      "logps/chosen": -421.876708984375,
+      "logps/rejected": -400.15728759765625,
+      "loss": 0.4724,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0339548587799072,
+      "rewards/margins": 0.9418285489082336,
+      "rewards/rejected": -1.9757835865020752,
       "step": 160
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 3.604108797288461e-07,
-      "logits/chosen": -2.2612414360046387,
-      "logits/rejected": -2.178300142288208,
-      "logps/chosen": -512.8599853515625,
-      "logps/rejected": -435.66693115234375,
-      "loss": 0.0375,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.0879271030426025,
-      "rewards/margins": 2.0758705139160156,
-      "rewards/rejected": -3.163797616958618,
+      "epoch": 0.19,
+      "learning_rate": 4.872702970909464e-07,
+      "logits/chosen": -2.2731730937957764,
+      "logits/rejected": -2.218116283416748,
+      "logps/chosen": -405.0645446777344,
+      "logps/rejected": -364.96832275390625,
+      "loss": 0.4362,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.658772349357605,
+      "rewards/margins": 1.0431811809539795,
+      "rewards/rejected": -1.7019535303115845,
       "step": 170
     },
     {
-      "epoch": 0.44,
-      "learning_rate": 3.40665367563858e-07,
-      "logits/chosen": -2.242908477783203,
-      "logits/rejected": -2.1256449222564697,
-      "logps/chosen": -503.54547119140625,
-      "logps/rejected": -403.26593017578125,
-      "loss": 0.037,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.299680471420288,
-      "rewards/margins": 1.7938661575317383,
-      "rewards/rejected": -3.0935468673706055,
+      "epoch": 0.2,
+      "learning_rate": 4.839692218542131e-07,
+      "logits/chosen": -2.172924757003784,
+      "logits/rejected": -2.1244688034057617,
+      "logps/chosen": -403.5210266113281,
+      "logps/rejected": -400.0758361816406,
+      "loss": 0.4438,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.9747382998466492,
+      "rewards/margins": 1.138419508934021,
+      "rewards/rejected": -2.1131579875946045,
       "step": 180
     },
     {
-      "epoch": 0.47,
-      "learning_rate": 3.202449097526798e-07,
-      "logits/chosen": -2.2713847160339355,
-      "logits/rejected": -2.163483142852783,
-      "logps/chosen": -501.21185302734375,
-      "logps/rejected": -417.178955078125,
-      "loss": 0.0478,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -1.1433241367340088,
-      "rewards/margins": 1.9010562896728516,
-      "rewards/rejected": -3.0443806648254395,
+      "epoch": 0.22,
+      "learning_rate": 4.803019108026997e-07,
+      "logits/chosen": -2.0333516597747803,
+      "logits/rejected": -2.007887840270996,
+      "logps/chosen": -449.2581481933594,
+      "logps/rejected": -425.9677734375,
+      "loss": 0.4446,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.2398478984832764,
+      "rewards/margins": 1.2218430042266846,
+      "rewards/rejected": -2.46169114112854,
       "step": 190
     },
     {
-      "epoch": 0.49,
-      "learning_rate": 2.993015235369905e-07,
-      "logits/chosen": -2.2553932666778564,
-      "logits/rejected": -2.1447877883911133,
-      "logps/chosen": -530.4608154296875,
-      "logps/rejected": -431.3677673339844,
-      "loss": 0.0435,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -0.9609147906303406,
-      "rewards/margins": 2.1664257049560547,
-      "rewards/rejected": -3.12734055519104,
+      "epoch": 0.23,
+      "learning_rate": 4.7627410443782887e-07,
+      "logits/chosen": -2.0739877223968506,
+      "logits/rejected": -2.035784959793091,
+      "logps/chosen": -468.93829345703125,
+      "logps/rejected": -442.99249267578125,
+      "loss": 0.4709,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.4709949493408203,
+      "rewards/margins": 0.9461126327514648,
+      "rewards/rejected": -2.417107582092285,
       "step": 200
     },
     {
-      "epoch": 0.49,
-      "eval_logits/chosen": -2.1734278202056885,
-      "eval_logits/rejected": -2.142083168029785,
-      "eval_logps/chosen": -475.05853271484375,
-      "eval_logps/rejected": -512.2731323242188,
-      "eval_loss": 0.07987947016954422,
-      "eval_rewards/accuracies": 0.62109375,
-      "eval_rewards/chosen": -2.1801888942718506,
-      "eval_rewards/margins": 0.36900976300239563,
-      "eval_rewards/rejected": -2.549198627471924,
-      "eval_runtime": 53.3455,
-      "eval_samples_per_second": 37.491,
-      "eval_steps_per_second": 0.6,
+      "epoch": 0.23,
+      "eval_logits/chosen": -2.172028064727783,
+      "eval_logits/rejected": -2.151461601257324,
+      "eval_logps/chosen": -370.87158203125,
+      "eval_logps/rejected": -433.51214599609375,
+      "eval_loss": 0.560804545879364,
+      "eval_rewards/accuracies": 0.7109375,
+      "eval_rewards/chosen": -1.138319492340088,
+      "eval_rewards/margins": 0.6232693195343018,
+      "eval_rewards/rejected": -1.7615886926651,
+      "eval_runtime": 52.9918,
+      "eval_samples_per_second": 37.742,
+      "eval_steps_per_second": 0.604,
       "step": 200
     },
     {
-      "epoch": 0.52,
-      "learning_rate": 2.7799111902582693e-07,
-      "logits/chosen": -2.1741156578063965,
-      "logits/rejected": -2.0635221004486084,
-      "logps/chosen": -524.964111328125,
-      "logps/rejected": -450.088623046875,
-      "loss": 0.0342,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.4079291820526123,
-      "rewards/margins": 1.9792110919952393,
-      "rewards/rejected": -3.3871402740478516,
+      "epoch": 0.24,
+      "learning_rate": 4.7189210755018034e-07,
+      "logits/chosen": -2.178704261779785,
+      "logits/rejected": -2.1404006481170654,
+      "logps/chosen": -413.65814208984375,
+      "logps/rejected": -374.3295593261719,
+      "loss": 0.4607,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.8382428884506226,
+      "rewards/margins": 1.006833791732788,
+      "rewards/rejected": -1.8450767993927002,
       "step": 210
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 2.564723385445869e-07,
-      "logits/chosen": -2.1615562438964844,
-      "logits/rejected": -2.0492231845855713,
-      "logps/chosen": -561.4410400390625,
-      "logps/rejected": -483.4110412597656,
-      "loss": 0.0356,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -1.2221920490264893,
-      "rewards/margins": 2.281812906265259,
-      "rewards/rejected": -3.504004955291748,
+      "epoch": 0.25,
+      "learning_rate": 4.671627793504988e-07,
+      "logits/chosen": -2.1911635398864746,
+      "logits/rejected": -2.098259210586548,
+      "logps/chosen": -445.24298095703125,
+      "logps/rejected": -407.4277038574219,
+      "loss": 0.4574,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.1178652048110962,
+      "rewards/margins": 1.1009081602096558,
+      "rewards/rejected": -2.218773365020752,
       "step": 220
     },
     {
-      "epoch": 0.57,
-      "learning_rate": 2.3490537564442845e-07,
-      "logits/chosen": -2.1855053901672363,
-      "logits/rejected": -2.075396776199341,
-      "logps/chosen": -530.4036254882812,
-      "logps/rejected": -457.735107421875,
-      "loss": 0.0301,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.2399654388427734,
-      "rewards/margins": 2.154582977294922,
-      "rewards/rejected": -3.3945488929748535,
+      "epoch": 0.26,
+      "learning_rate": 4.6209352273286095e-07,
+      "logits/chosen": -2.0867063999176025,
+      "logits/rejected": -2.0185961723327637,
+      "logps/chosen": -440.40093994140625,
+      "logps/rejected": -437.75811767578125,
+      "loss": 0.4422,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -1.0934244394302368,
+      "rewards/margins": 1.364727258682251,
+      "rewards/rejected": -2.4581520557403564,
       "step": 230
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 2.1345078256378801e-07,
-      "logits/chosen": -2.187288284301758,
-      "logits/rejected": -2.0610218048095703,
-      "logps/chosen": -533.1801147460938,
-      "logps/rejected": -487.4967346191406,
-      "loss": 0.0244,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -1.2845171689987183,
-      "rewards/margins": 2.3898768424987793,
-      "rewards/rejected": -3.674394130706787,
+      "epoch": 0.27,
+      "learning_rate": 4.56692272686805e-07,
+      "logits/chosen": -2.0416159629821777,
+      "logits/rejected": -1.9667298793792725,
+      "logps/chosen": -497.56610107421875,
+      "logps/rejected": -434.28863525390625,
+      "loss": 0.4247,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.3035621643066406,
+      "rewards/margins": 1.2145717144012451,
+      "rewards/rejected": -2.5181336402893066,
       "step": 240
     },
     {
-      "epoch": 0.62,
-      "learning_rate": 1.9226827501969865e-07,
-      "logits/chosen": -2.166536331176758,
-      "logits/rejected": -2.0207314491271973,
-      "logps/chosen": -566.1360473632812,
-      "logps/rejected": -520.3924560546875,
-      "loss": 0.0354,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": -1.3790395259857178,
-      "rewards/margins": 2.647700786590576,
-      "rewards/rejected": -4.026740550994873,
+      "epoch": 0.28,
+      "learning_rate": 4.5096748387656326e-07,
+      "logits/chosen": -2.1263952255249023,
+      "logits/rejected": -2.0554358959198,
+      "logps/chosen": -489.86602783203125,
+      "logps/rejected": -454.3324279785156,
+      "loss": 0.4466,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.5867921113967896,
+      "rewards/margins": 1.2441270351409912,
+      "rewards/rejected": -2.830918788909912,
       "step": 250
     },
     {
-      "epoch": 0.64,
-      "learning_rate": 1.715155432264775e-07,
-      "logits/chosen": -2.160069704055786,
-      "logits/rejected": -2.0387001037597656,
-      "logps/chosen": -520.0031127929688,
-      "logps/rejected": -447.298828125,
-      "loss": 0.0421,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.3448396921157837,
-      "rewards/margins": 2.034497022628784,
-      "rewards/rejected": -3.3793368339538574,
+      "epoch": 0.29,
+      "learning_rate": 4.4492811740683877e-07,
+      "logits/chosen": -1.9983489513397217,
+      "logits/rejected": -1.9614261388778687,
+      "logps/chosen": -488.27056884765625,
+      "logps/rejected": -473.0369567871094,
+      "loss": 0.4341,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.4624744653701782,
+      "rewards/margins": 1.2226989269256592,
+      "rewards/rejected": -2.685173511505127,
       "step": 260
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 1.51347077992983e-07,
-      "logits/chosen": -2.201711416244507,
-      "logits/rejected": -2.0760598182678223,
-      "logps/chosen": -518.4276733398438,
-      "logps/rejected": -448.0538024902344,
-      "loss": 0.0352,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -1.2280199527740479,
-      "rewards/margins": 2.027623176574707,
-      "rewards/rejected": -3.255643129348755,
+      "epoch": 0.31,
+      "learning_rate": 4.3858362679584354e-07,
+      "logits/chosen": -2.133930206298828,
+      "logits/rejected": -2.0397889614105225,
+      "logps/chosen": -483.59979248046875,
+      "logps/rejected": -459.24053955078125,
+      "loss": 0.4415,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.6541706323623657,
+      "rewards/margins": 1.0623505115509033,
+      "rewards/rejected": -2.7165210247039795,
       "step": 270
     },
     {
-      "epoch": 0.69,
-      "learning_rate": 1.3191302063739906e-07,
-      "logits/chosen": -2.2061142921447754,
-      "logits/rejected": -2.08331298828125,
-      "logps/chosen": -536.6764526367188,
-      "logps/rejected": -460.00250244140625,
-      "loss": 0.0335,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.3899211883544922,
-      "rewards/margins": 2.058790445327759,
-      "rewards/rejected": -3.44871187210083,
+      "epoch": 0.32,
+      "learning_rate": 4.3194394317755245e-07,
+      "logits/chosen": -2.099963426589966,
+      "logits/rejected": -2.0109364986419678,
+      "logps/chosen": -519.8792724609375,
+      "logps/rejected": -501.48187255859375,
+      "loss": 0.4303,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.325263261795044,
+      "rewards/margins": 1.4841763973236084,
+      "rewards/rejected": -2.8094398975372314,
       "step": 280
     },
     {
-      "epoch": 0.72,
-      "learning_rate": 1.1335804528119475e-07,
-      "logits/chosen": -2.194516658782959,
-      "logits/rejected": -2.0347750186920166,
-      "logps/chosen": -583.8428955078125,
-      "logps/rejected": -485.03057861328125,
-      "loss": 0.0266,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.3776942491531372,
-      "rewards/margins": 2.384459972381592,
-      "rewards/rejected": -3.7621541023254395,
+      "epoch": 0.33,
+      "learning_rate": 4.2501945975633914e-07,
+      "logits/chosen": -2.1019272804260254,
+      "logits/rejected": -1.9885908365249634,
+      "logps/chosen": -519.796630859375,
+      "logps/rejected": -514.5078125,
+      "loss": 0.4384,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.6320844888687134,
+      "rewards/margins": 1.3468807935714722,
+      "rewards/rejected": -2.9789652824401855,
       "step": 290
     },
     {
-      "epoch": 0.74,
-      "learning_rate": 9.582028184286423e-08,
-      "logits/chosen": -2.1787655353546143,
-      "logits/rejected": -2.075291156768799,
-      "logps/chosen": -514.74072265625,
-      "logps/rejected": -461.1167907714844,
-      "loss": 0.0288,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.5570707321166992,
-      "rewards/margins": 1.984126091003418,
-      "rewards/rejected": -3.541196823120117,
+      "epoch": 0.34,
+      "learning_rate": 4.1782101553832405e-07,
+      "logits/chosen": -2.0356638431549072,
+      "logits/rejected": -1.985108733177185,
+      "logps/chosen": -473.4766540527344,
+      "logps/rejected": -430.5616149902344,
+      "loss": 0.4289,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.208434820175171,
+      "rewards/margins": 1.417704701423645,
+      "rewards/rejected": -2.6261394023895264,
       "step": 300
     },
     {
-      "epoch": 0.74,
-      "eval_logits/chosen": -2.1675052642822266,
-      "eval_logits/rejected": -2.133922576904297,
-      "eval_logps/chosen": -500.8697204589844,
-      "eval_logps/rejected": -548.4016723632812,
-      "eval_loss": 0.07103094458580017,
-      "eval_rewards/accuracies": 0.6171875,
-      "eval_rewards/chosen": -2.438300848007202,
-      "eval_rewards/margins": 0.4721827805042267,
-      "eval_rewards/rejected": -2.9104835987091064,
-      "eval_runtime": 53.3323,
-      "eval_samples_per_second": 37.501,
-      "eval_steps_per_second": 0.6,
+      "epoch": 0.34,
+      "eval_logits/chosen": -2.1203949451446533,
+      "eval_logits/rejected": -2.0882043838500977,
+      "eval_logps/chosen": -411.08111572265625,
+      "eval_logps/rejected": -496.93798828125,
+      "eval_loss": 0.5292558670043945,
+      "eval_rewards/accuracies": 0.75390625,
+      "eval_rewards/chosen": -1.5404143333435059,
+      "eval_rewards/margins": 0.8554330468177795,
+      "eval_rewards/rejected": -2.3958473205566406,
+      "eval_runtime": 52.9556,
+      "eval_samples_per_second": 37.767,
+      "eval_steps_per_second": 0.604,
       "step": 300
     },
     {
-      "epoch": 0.76,
-      "learning_rate": 7.943028774907065e-08,
-      "logits/chosen": -2.1723272800445557,
-      "logits/rejected": -2.0614748001098633,
-      "logps/chosen": -535.1619873046875,
-      "logps/rejected": -455.0377502441406,
-      "loss": 0.0292,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.294924020767212,
-      "rewards/margins": 2.182978630065918,
-      "rewards/rejected": -3.47790265083313,
+      "epoch": 0.35,
+      "learning_rate": 4.103598783649029e-07,
+      "logits/chosen": -2.0809109210968018,
+      "logits/rejected": -2.0090012550354004,
+      "logps/chosen": -435.18414306640625,
+      "logps/rejected": -414.2064514160156,
+      "loss": 0.4236,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.0976500511169434,
+      "rewards/margins": 1.1829910278320312,
+      "rewards/rejected": -2.2806410789489746,
       "step": 310
     },
     {
-      "epoch": 0.79,
-      "learning_rate": 6.431007601814637e-08,
-      "logits/chosen": -2.2771997451782227,
-      "logits/rejected": -2.146629810333252,
-      "logps/chosen": -576.0748291015625,
-      "logps/rejected": -492.65167236328125,
-      "loss": 0.031,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.481297254562378,
-      "rewards/margins": 2.240931510925293,
-      "rewards/rejected": -3.722228527069092,
+      "epoch": 0.36,
+      "learning_rate": 4.026477272750119e-07,
+      "logits/chosen": -2.0164308547973633,
+      "logits/rejected": -1.9634422063827515,
+      "logps/chosen": -502.15911865234375,
+      "logps/rejected": -492.40350341796875,
+      "loss": 0.454,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.72931706905365,
+      "rewards/margins": 1.1704776287078857,
+      "rewards/rejected": -2.8997950553894043,
       "step": 320
     },
     {
-      "epoch": 0.81,
-      "learning_rate": 5.0572206951246e-08,
-      "logits/chosen": -2.2158615589141846,
-      "logits/rejected": -2.0619277954101562,
-      "logps/chosen": -581.7293701171875,
-      "logps/rejected": -495.04638671875,
-      "loss": 0.0317,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": -1.2919968366622925,
-      "rewards/margins": 2.5048012733459473,
-      "rewards/rejected": -3.7967982292175293,
+      "epoch": 0.37,
+      "learning_rate": 3.9469663422373864e-07,
+      "logits/chosen": -2.0206305980682373,
+      "logits/rejected": -1.9137179851531982,
+      "logps/chosen": -533.25537109375,
+      "logps/rejected": -506.52899169921875,
+      "loss": 0.4423,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.628199815750122,
+      "rewards/margins": 1.3168383836746216,
+      "rewards/rejected": -2.945038318634033,
       "step": 330
     },
     {
-      "epoch": 0.84,
-      "learning_rate": 3.831895019292897e-08,
-      "logits/chosen": -2.2480924129486084,
-      "logits/rejected": -2.110482692718506,
-      "logps/chosen": -597.642333984375,
-      "logps/rejected": -521.8727416992188,
-      "loss": 0.0281,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -1.3266541957855225,
-      "rewards/margins": 2.570845127105713,
-      "rewards/rejected": -3.8974990844726562,
+      "epoch": 0.38,
+      "learning_rate": 3.865190451858954e-07,
+      "logits/chosen": -1.9749759435653687,
+      "logits/rejected": -1.9049227237701416,
+      "logps/chosen": -466.3131408691406,
+      "logps/rejected": -443.3275451660156,
+      "loss": 0.4345,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.4465491771697998,
+      "rewards/margins": 1.1582015752792358,
+      "rewards/rejected": -2.604750394821167,
       "step": 340
     },
     {
-      "epoch": 0.86,
-      "learning_rate": 2.764152339909756e-08,
-      "logits/chosen": -2.1906208992004395,
-      "logits/rejected": -2.0595510005950928,
-      "logps/chosen": -539.5335083007812,
-      "logps/rejected": -462.76025390625,
-      "loss": 0.034,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.460292935371399,
-      "rewards/margins": 2.1251232624053955,
-      "rewards/rejected": -3.585416078567505,
+      "epoch": 0.4,
+      "learning_rate": 3.781277606741327e-07,
+      "logits/chosen": -1.8896055221557617,
+      "logits/rejected": -1.6728490591049194,
+      "logps/chosen": -479.135986328125,
+      "logps/rejected": -470.610107421875,
+      "loss": 0.4149,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -1.2967784404754639,
+      "rewards/margins": 1.580676794052124,
+      "rewards/rejected": -2.877455234527588,
       "step": 350
     },
     {
-      "epoch": 0.89,
-      "learning_rate": 1.861941317991664e-08,
-      "logits/chosen": -2.1756224632263184,
-      "logits/rejected": -2.0511136054992676,
-      "logps/chosen": -544.6201171875,
-      "logps/rejected": -468.48760986328125,
-      "loss": 0.0303,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.5998585224151611,
-      "rewards/margins": 1.8565304279327393,
-      "rewards/rejected": -3.4563891887664795,
+      "epoch": 0.41,
+      "learning_rate": 3.6953591570208996e-07,
+      "logits/chosen": -1.6949100494384766,
+      "logits/rejected": -1.4142816066741943,
+      "logps/chosen": -497.4435119628906,
+      "logps/rejected": -502.25799560546875,
+      "loss": 0.439,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -1.900599479675293,
+      "rewards/margins": 1.1688846349716187,
+      "rewards/rejected": -3.069484233856201,
       "step": 360
     },
     {
-      "epoch": 0.91,
-      "learning_rate": 1.13197833728636e-08,
-      "logits/chosen": -2.1858394145965576,
-      "logits/rejected": -2.0600364208221436,
-      "logps/chosen": -554.5750732421875,
-      "logps/rejected": -469.27484130859375,
-      "loss": 0.0296,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.3729755878448486,
-      "rewards/margins": 2.1774544715881348,
-      "rewards/rejected": -3.5504302978515625,
+      "epoch": 0.42,
+      "learning_rate": 3.607569592239452e-07,
+      "logits/chosen": -1.7967230081558228,
+      "logits/rejected": -1.6560561656951904,
+      "logps/chosen": -447.05877685546875,
+      "logps/rejected": -430.55865478515625,
+      "loss": 0.4419,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.2963721752166748,
+      "rewards/margins": 1.0919477939605713,
+      "rewards/rejected": -2.388319730758667,
       "step": 370
     },
     {
-      "epoch": 0.94,
-      "learning_rate": 5.79697505093521e-09,
-      "logits/chosen": -2.161980628967285,
-      "logits/rejected": -2.0485546588897705,
-      "logps/chosen": -514.3355712890625,
-      "logps/rejected": -432.40704345703125,
-      "loss": 0.0282,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.297263741493225,
-      "rewards/margins": 1.9683955907821655,
-      "rewards/rejected": -3.2656593322753906,
+      "epoch": 0.43,
+      "learning_rate": 3.518046330825494e-07,
+      "logits/chosen": -1.5726587772369385,
+      "logits/rejected": -1.3357789516448975,
+      "logps/chosen": -474.6183166503906,
+      "logps/rejected": -467.42144775390625,
+      "loss": 0.4082,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.395471453666687,
+      "rewards/margins": 1.4275726079940796,
+      "rewards/rejected": -2.8230443000793457,
       "step": 380
     },
     {
-      "epoch": 0.96,
-      "learning_rate": 2.092101988131256e-09,
-      "logits/chosen": -2.2410452365875244,
-      "logits/rejected": -2.109318256378174,
-      "logps/chosen": -566.7930297851562,
-      "logps/rejected": -473.2035217285156,
-      "loss": 0.0274,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.3028382062911987,
-      "rewards/margins": 2.2111973762512207,
-      "rewards/rejected": -3.514035701751709,
+      "epoch": 0.44,
+      "learning_rate": 3.4269295049909713e-07,
+      "logits/chosen": -1.300771951675415,
+      "logits/rejected": -1.0319937467575073,
+      "logps/chosen": -538.4091796875,
+      "logps/rejected": -550.3985595703125,
+      "loss": 0.4245,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.087444305419922,
+      "rewards/margins": 1.5177741050720215,
+      "rewards/rejected": -3.6052188873291016,
       "step": 390
     },
     {
-      "epoch": 0.99,
-      "learning_rate": 2.327445937151673e-10,
-      "logits/chosen": -2.168402910232544,
-      "logits/rejected": -2.057384967803955,
-      "logps/chosen": -555.5765380859375,
-      "logps/rejected": -468.2821350097656,
-      "loss": 0.032,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -1.3704006671905518,
-      "rewards/margins": 2.0777838230133057,
-      "rewards/rejected": -3.4481842517852783,
+      "epoch": 0.45,
+      "learning_rate": 3.3343617413800453e-07,
+      "logits/chosen": -1.3637840747833252,
+      "logits/rejected": -1.0050981044769287,
+      "logps/chosen": -517.8087768554688,
+      "logps/rejected": -541.1546020507812,
+      "loss": 0.4195,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8266891241073608,
+      "rewards/margins": 1.5495359897613525,
+      "rewards/rejected": -3.3762245178222656,
       "step": 400
     },
     {
-      "epoch": 0.99,
-      "eval_logits/chosen": -2.163764476776123,
-      "eval_logits/rejected": -2.129502534866333,
-      "eval_logps/chosen": -504.9170227050781,
-      "eval_logps/rejected": -552.4011840820312,
-      "eval_loss": 0.06752217561006546,
-      "eval_rewards/accuracies": 0.640625,
-      "eval_rewards/chosen": -2.478773355484009,
-      "eval_rewards/margins": 0.47170597314834595,
-      "eval_rewards/rejected": -2.950479507446289,
-      "eval_runtime": 53.3856,
-      "eval_samples_per_second": 37.463,
-      "eval_steps_per_second": 0.599,
+      "epoch": 0.45,
+      "eval_logits/chosen": -1.297582983970642,
+      "eval_logits/rejected": -1.0570608377456665,
+      "eval_logps/chosen": -436.197021484375,
+      "eval_logps/rejected": -547.3040771484375,
+      "eval_loss": 0.509623646736145,
+      "eval_rewards/accuracies": 0.78125,
+      "eval_rewards/chosen": -1.7915737628936768,
+      "eval_rewards/margins": 1.1079347133636475,
+      "eval_rewards/rejected": -2.899508476257324,
+      "eval_runtime": 52.9183,
+      "eval_samples_per_second": 37.794,
+      "eval_steps_per_second": 0.605,
       "step": 400
     },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2404879378132893e-07,
+      "logits/chosen": -1.4247735738754272,
+      "logits/rejected": -1.0035231113433838,
+      "logps/chosen": -466.69964599609375,
+      "logps/rejected": -445.894287109375,
+      "loss": 0.415,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.3474479913711548,
+      "rewards/margins": 1.4210073947906494,
+      "rewards/rejected": -2.768455743789673,
+      "step": 410
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1454550364767894e-07,
+      "logits/chosen": -1.5311955213546753,
+      "logits/rejected": -1.2638423442840576,
+      "logps/chosen": -522.9796142578125,
+      "logps/rejected": -500.9322814941406,
+      "loss": 0.4131,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.5469942092895508,
+      "rewards/margins": 1.44197416305542,
+      "rewards/rejected": -2.9889683723449707,
+      "step": 420
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.049411793911154e-07,
+      "logits/chosen": -1.5343776941299438,
+      "logits/rejected": -1.0935866832733154,
+      "logps/chosen": -515.9620971679688,
+      "logps/rejected": -521.0631103515625,
+      "loss": 0.4101,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.6215946674346924,
+      "rewards/margins": 1.5825344324111938,
+      "rewards/rejected": -3.204129457473755,
+      "step": 430
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9525085481604914e-07,
+      "logits/chosen": -1.2731831073760986,
+      "logits/rejected": -0.8194789886474609,
+      "logps/chosen": -521.5642700195312,
+      "logps/rejected": -497.09075927734375,
+      "loss": 0.4035,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -1.4858245849609375,
+      "rewards/margins": 1.8156074285507202,
+      "rewards/rejected": -3.301431655883789,
+      "step": 440
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.854896983445833e-07,
+      "logits/chosen": -1.4735174179077148,
+      "logits/rejected": -1.1519749164581299,
+      "logps/chosen": -523.7742919921875,
+      "logps/rejected": -497.34075927734375,
+      "loss": 0.4077,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.5442702770233154,
+      "rewards/margins": 1.5848388671875,
+      "rewards/rejected": -3.1291093826293945,
+      "step": 450
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7567298927313654e-07,
+      "logits/chosen": -1.3687413930892944,
+      "logits/rejected": -1.104038953781128,
+      "logps/chosen": -545.3075561523438,
+      "logps/rejected": -563.0989990234375,
+      "loss": 0.4223,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.9977827072143555,
+      "rewards/margins": 1.3200435638427734,
+      "rewards/rejected": -3.317826509475708,
+      "step": 460
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.658160938555123e-07,
+      "logits/chosen": -1.1723263263702393,
+      "logits/rejected": -0.8053513765335083,
+      "logps/chosen": -516.1744995117188,
+      "logps/rejected": -529.2353515625,
+      "loss": 0.4353,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.1037230491638184,
+      "rewards/margins": 1.373581886291504,
+      "rewards/rejected": -3.4773049354553223,
+      "step": 470
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.559344412498532e-07,
+      "logits/chosen": -1.3750767707824707,
+      "logits/rejected": -1.1043331623077393,
+      "logps/chosen": -494.09619140625,
+      "logps/rejected": -510.038818359375,
+      "loss": 0.4033,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -1.6110206842422485,
+      "rewards/margins": 1.3316563367843628,
+      "rewards/rejected": -2.9426772594451904,
+      "step": 480
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.460434993671294e-07,
+      "logits/chosen": -1.3292906284332275,
+      "logits/rejected": -0.969641387462616,
+      "logps/chosen": -499.32977294921875,
+      "logps/rejected": -508.908447265625,
+      "loss": 0.4074,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8106292486190796,
+      "rewards/margins": 1.3505693674087524,
+      "rewards/rejected": -3.161198616027832,
+      "step": 490
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.361587506589672e-07,
+      "logits/chosen": -1.271968126296997,
+      "logits/rejected": -0.9058429002761841,
+      "logps/chosen": -505.27294921875,
+      "logps/rejected": -511.2696228027344,
+      "loss": 0.3891,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -1.695939064025879,
+      "rewards/margins": 1.5928833484649658,
+      "rewards/rejected": -3.288822650909424,
+      "step": 500
+    },
+    {
+      "epoch": 0.57,
+      "eval_logits/chosen": -1.1313989162445068,
+      "eval_logits/rejected": -0.8608421087265015,
+      "eval_logps/chosen": -517.5072021484375,
+      "eval_logps/rejected": -649.901611328125,
+      "eval_loss": 0.5085692405700684,
+      "eval_rewards/accuracies": 0.78125,
+      "eval_rewards/chosen": -2.60467529296875,
+      "eval_rewards/margins": 1.3208080530166626,
+      "eval_rewards/rejected": -3.925483226776123,
+      "eval_runtime": 52.952,
+      "eval_samples_per_second": 37.77,
+      "eval_steps_per_second": 0.604,
+      "step": 500
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2629566788271613e-07,
+      "logits/chosen": -1.1410466432571411,
+      "logits/rejected": -0.7384732365608215,
+      "logps/chosen": -585.402099609375,
+      "logps/rejected": -608.8575439453125,
+      "loss": 0.402,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.134608268737793,
+      "rewards/margins": 2.0132174491882324,
+      "rewards/rejected": -4.147825717926025,
+      "step": 510
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1646968988169135e-07,
+      "logits/chosen": -1.249940276145935,
+      "logits/rejected": -0.7865885496139526,
+      "logps/chosen": -539.4969482421875,
+      "logps/rejected": -514.3495483398438,
+      "loss": 0.3732,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -1.8132226467132568,
+      "rewards/margins": 1.6672155857086182,
+      "rewards/rejected": -3.480438232421875,
+      "step": 520
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0669619741850232e-07,
+      "logits/chosen": -1.2208908796310425,
+      "logits/rejected": -0.6407105922698975,
+      "logps/chosen": -574.2348022460938,
+      "logps/rejected": -547.8206787109375,
+      "loss": 0.3974,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.831973671913147,
+      "rewards/margins": 1.7614845037460327,
+      "rewards/rejected": -3.5934581756591797,
+      "step": 530
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9699048909929518e-07,
+      "logits/chosen": -1.1419695615768433,
+      "logits/rejected": -0.7198764681816101,
+      "logps/chosen": -544.6316528320312,
+      "logps/rejected": -555.2297973632812,
+      "loss": 0.4004,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -1.7934131622314453,
+      "rewards/margins": 1.6694806814193726,
+      "rewards/rejected": -3.4628939628601074,
+      "step": 540
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8736775742659732e-07,
+      "logits/chosen": -1.162408471107483,
+      "logits/rejected": -0.6814004182815552,
+      "logps/chosen": -485.46234130859375,
+      "logps/rejected": -522.3754272460938,
+      "loss": 0.3946,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.6819603443145752,
+      "rewards/margins": 1.7173573970794678,
+      "rewards/rejected": -3.399317979812622,
+      "step": 550
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7784306501824616e-07,
+      "logits/chosen": -1.2207626104354858,
+      "logits/rejected": -0.8465560674667358,
+      "logps/chosen": -520.7455444335938,
+      "logps/rejected": -532.8163452148438,
+      "loss": 0.3927,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -1.7277084589004517,
+      "rewards/margins": 1.5865801572799683,
+      "rewards/rejected": -3.31428861618042,
+      "step": 560
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6843132102963025e-07,
+      "logits/chosen": -1.1111127138137817,
+      "logits/rejected": -0.8249386548995972,
+      "logps/chosen": -499.12030029296875,
+      "logps/rejected": -508.18719482421875,
+      "loss": 0.4308,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -1.8714163303375244,
+      "rewards/margins": 1.3305481672286987,
+      "rewards/rejected": -3.2019646167755127,
+      "step": 570
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.591472578161458e-07,
+      "logits/chosen": -1.1796176433563232,
+      "logits/rejected": -0.8019029498100281,
+      "logps/chosen": -501.35992431640625,
+      "logps/rejected": -508.736572265625,
+      "loss": 0.3972,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -1.7980735301971436,
+      "rewards/margins": 1.468266248703003,
+      "rewards/rejected": -3.2663397789001465,
+      "step": 580
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5000540787240274e-07,
+      "logits/chosen": -0.9903499484062195,
+      "logits/rejected": -0.6644443273544312,
+      "logps/chosen": -504.76409912109375,
+      "logps/rejected": -518.8839721679688,
+      "loss": 0.3749,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.8273483514785767,
+      "rewards/margins": 1.4342434406280518,
+      "rewards/rejected": -3.261591672897339,
+      "step": 590
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.410200810842749e-07,
+      "logits/chosen": -0.9152101278305054,
+      "logits/rejected": -0.43462666869163513,
+      "logps/chosen": -494.7369689941406,
+      "logps/rejected": -552.8912353515625,
+      "loss": 0.4182,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.983428955078125,
+      "rewards/margins": 1.5005247592926025,
+      "rewards/rejected": -3.4839534759521484,
+      "step": 600
+    },
+    {
+      "epoch": 0.68,
+      "eval_logits/chosen": -0.838397741317749,
+      "eval_logits/rejected": -0.4354328513145447,
+      "eval_logps/chosen": -506.7194519042969,
+      "eval_logps/rejected": -636.9742431640625,
+      "eval_loss": 0.49758800864219666,
+      "eval_rewards/accuracies": 0.76953125,
+      "eval_rewards/chosen": -2.496797561645508,
+      "eval_rewards/margins": 1.2994122505187988,
+      "eval_rewards/rejected": -3.7962098121643066,
+      "eval_runtime": 52.9477,
+      "eval_samples_per_second": 37.773,
+      "eval_steps_per_second": 0.604,
+      "step": 600
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.322053423294041e-07,
+      "logits/chosen": -0.7785909175872803,
+      "logits/rejected": -0.1633441299200058,
+      "logps/chosen": -553.189697265625,
+      "logps/rejected": -588.4357299804688,
+      "loss": 0.386,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.27974009513855,
+      "rewards/margins": 1.8430849313735962,
+      "rewards/rejected": -4.122824668884277,
+      "step": 610
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2357498946121905e-07,
+      "logits/chosen": -0.7135148644447327,
+      "logits/rejected": -0.3086184859275818,
+      "logps/chosen": -552.1744384765625,
+      "logps/rejected": -593.6995849609375,
+      "loss": 0.3885,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.253349781036377,
+      "rewards/margins": 1.7079604864120483,
+      "rewards/rejected": -3.9613101482391357,
+      "step": 620
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1514253171093161e-07,
+      "logits/chosen": -0.8449075818061829,
+      "logits/rejected": -0.3063100278377533,
+      "logps/chosen": -538.8817749023438,
+      "logps/rejected": -539.5289306640625,
+      "loss": 0.4102,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.042537212371826,
+      "rewards/margins": 1.56886887550354,
+      "rewards/rejected": -3.611405611038208,
+      "step": 630
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0692116854131883e-07,
+      "logits/chosen": -0.7637182474136353,
+      "logits/rejected": -0.20512516796588898,
+      "logps/chosen": -576.058349609375,
+      "logps/rejected": -590.4248046875,
+      "loss": 0.4046,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.343686580657959,
+      "rewards/margins": 1.6815674304962158,
+      "rewards/rejected": -4.025254249572754,
+      "step": 640
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.89237689853889e-08,
+      "logits/chosen": -0.8669427633285522,
+      "logits/rejected": -0.28752464056015015,
+      "logps/chosen": -585.344482421875,
+      "logps/rejected": -613.337646484375,
+      "loss": 0.3747,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.232487440109253,
+      "rewards/margins": 1.73616623878479,
+      "rewards/rejected": -3.968653917312622,
+      "step": 650
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.11628515022765e-08,
+      "logits/chosen": -0.8596148490905762,
+      "logits/rejected": -0.15762929618358612,
+      "logps/chosen": -537.1088256835938,
+      "logps/rejected": -582.6017456054688,
+      "loss": 0.3997,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.121387481689453,
+      "rewards/margins": 1.9997097253799438,
+      "rewards/rejected": -4.121097087860107,
+      "step": 660
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.365056438189486e-08,
+      "logits/chosen": -0.9106462597846985,
+      "logits/rejected": -0.3486366868019104,
+      "logps/chosen": -603.9283447265625,
+      "logps/rejected": -620.8770141601562,
+      "loss": 0.394,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.358280897140503,
+      "rewards/margins": 1.6525799036026,
+      "rewards/rejected": -4.010860919952393,
+      "step": 670
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.639866672902101e-08,
+      "logits/chosen": -0.8860856294631958,
+      "logits/rejected": -0.19501088559627533,
+      "logps/chosen": -558.152587890625,
+      "logps/rejected": -582.1339721679688,
+      "loss": 0.3877,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -2.326817750930786,
+      "rewards/margins": 1.7357206344604492,
+      "rewards/rejected": -4.0625386238098145,
+      "step": 680
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.941851005657851e-08,
+      "logits/chosen": -0.7948504686355591,
+      "logits/rejected": -0.22474519908428192,
+      "logps/chosen": -547.021484375,
+      "logps/rejected": -597.5274047851562,
+      "loss": 0.3979,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -2.1950807571411133,
+      "rewards/margins": 1.8335994482040405,
+      "rewards/rejected": -4.028680324554443,
+      "step": 690
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.272102051693051e-08,
+      "logits/chosen": -0.7106924057006836,
+      "logits/rejected": -0.11677990108728409,
+      "logps/chosen": -548.4378662109375,
+      "logps/rejected": -583.5083618164062,
+      "loss": 0.3845,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.1655876636505127,
+      "rewards/margins": 1.7359482049942017,
+      "rewards/rejected": -3.901536226272583,
+      "step": 700
+    },
+    {
+      "epoch": 0.79,
+      "eval_logits/chosen": -0.7200266718864441,
+      "eval_logits/rejected": -0.2825911045074463,
+      "eval_logps/chosen": -526.7998657226562,
+      "eval_logps/rejected": -658.1885375976562,
+      "eval_loss": 0.4966849386692047,
+      "eval_rewards/accuracies": 0.76953125,
+      "eval_rewards/chosen": -2.6976022720336914,
+      "eval_rewards/margins": 1.3107508420944214,
+      "eval_rewards/rejected": -4.008352756500244,
+      "eval_runtime": 52.9036,
+      "eval_samples_per_second": 37.805,
+      "eval_steps_per_second": 0.605,
+      "step": 700
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.6316681798995844e-08,
+      "logits/chosen": -0.7543665170669556,
+      "logits/rejected": -0.3065817952156067,
+      "logps/chosen": -570.3900146484375,
+      "logps/rejected": -601.1921997070312,
+      "loss": 0.3962,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.327669858932495,
+      "rewards/margins": 1.7061312198638916,
+      "rewards/rejected": -4.033801555633545,
+      "step": 710
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.0215518717961256e-08,
+      "logits/chosen": -0.8484760522842407,
+      "logits/rejected": -0.2536430358886719,
+      "logps/chosen": -571.7559204101562,
+      "logps/rejected": -592.8436889648438,
+      "loss": 0.3922,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -2.2247302532196045,
+      "rewards/margins": 1.9966415166854858,
+      "rewards/rejected": -4.221371650695801,
+      "step": 720
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.4427081523275925e-08,
+      "logits/chosen": -0.8367404937744141,
+      "logits/rejected": -0.18715055286884308,
+      "logps/chosen": -587.681884765625,
+      "logps/rejected": -586.436279296875,
+      "loss": 0.3943,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.551619291305542,
+      "rewards/margins": 1.5289732217788696,
+      "rewards/rejected": -4.080592155456543,
+      "step": 730
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.896043094949061e-08,
+      "logits/chosen": -0.7256150245666504,
+      "logits/rejected": -0.2999003529548645,
+      "logps/chosen": -577.1644897460938,
+      "logps/rejected": -619.8590698242188,
+      "loss": 0.3992,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.4170877933502197,
+      "rewards/margins": 1.7640550136566162,
+      "rewards/rejected": -4.181142330169678,
+      "step": 740
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3824124033343557e-08,
+      "logits/chosen": -0.7496400475502014,
+      "logits/rejected": -0.18662114441394806,
+      "logps/chosen": -579.8140869140625,
+      "logps/rejected": -607.1034545898438,
+      "loss": 0.4029,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -2.3119702339172363,
+      "rewards/margins": 1.9123681783676147,
+      "rewards/rejected": -4.224338531494141,
+      "step": 750
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9026200719291904e-08,
+      "logits/chosen": -0.8657780885696411,
+      "logits/rejected": -0.27606701850891113,
+      "logps/chosen": -584.1732177734375,
+      "logps/rejected": -619.30859375,
+      "loss": 0.3828,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.2997066974639893,
+      "rewards/margins": 1.865792989730835,
+      "rewards/rejected": -4.165499687194824,
+      "step": 760
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4574171274456433e-08,
+      "logits/chosen": -0.7996637225151062,
+      "logits/rejected": -0.18977174162864685,
+      "logps/chosen": -581.4425659179688,
+      "logps/rejected": -592.1394653320312,
+      "loss": 0.3729,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.2692534923553467,
+      "rewards/margins": 1.7808440923690796,
+      "rewards/rejected": -4.050097942352295,
+      "step": 770
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.047500453267881e-08,
+      "logits/chosen": -0.7119861245155334,
+      "logits/rejected": -0.11405928432941437,
+      "logps/chosen": -592.5960083007812,
+      "logps/rejected": -605.414306640625,
+      "loss": 0.399,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.3485360145568848,
+      "rewards/margins": 1.844259262084961,
+      "rewards/rejected": -4.192794322967529,
+      "step": 780
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.673511698609292e-08,
+      "logits/chosen": -0.7867909669876099,
+      "logits/rejected": -0.11047197878360748,
+      "logps/chosen": -590.2078857421875,
+      "logps/rejected": -598.898681640625,
+      "loss": 0.41,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -2.408952236175537,
+      "rewards/margins": 1.6835685968399048,
+      "rewards/rejected": -4.092520713806152,
+      "step": 790
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.3360362741285769e-08,
+      "logits/chosen": -0.7789919972419739,
+      "logits/rejected": -0.2865374684333801,
+      "logps/chosen": -587.2545166015625,
+      "logps/rejected": -613.9022216796875,
+      "loss": 0.3896,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.680772304534912,
+      "rewards/margins": 1.56209397315979,
+      "rewards/rejected": -4.242866516113281,
+      "step": 800
+    },
+    {
+      "epoch": 0.91,
+      "eval_logits/chosen": -0.6531276106834412,
+      "eval_logits/rejected": -0.1928076446056366,
+      "eval_logps/chosen": -554.1226196289062,
+      "eval_logps/rejected": -687.527099609375,
+      "eval_loss": 0.4964603781700134,
+      "eval_rewards/accuracies": 0.76953125,
+      "eval_rewards/chosen": -2.970829725265503,
+      "eval_rewards/margins": 1.3309086561203003,
+      "eval_rewards/rejected": -4.301738739013672,
+      "eval_runtime": 52.9306,
+      "eval_samples_per_second": 37.785,
+      "eval_steps_per_second": 0.605,
+      "step": 800
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0356024355769433e-08,
+      "logits/chosen": -0.7060940861701965,
+      "logits/rejected": -0.22113999724388123,
+      "logps/chosen": -593.3995361328125,
+      "logps/rejected": -653.0950317382812,
+      "loss": 0.3726,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -2.5047855377197266,
+      "rewards/margins": 1.8576644659042358,
+      "rewards/rejected": -4.36245059967041,
+      "step": 810
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.726804569108597e-09,
+      "logits/chosen": -0.7532386779785156,
+      "logits/rejected": -0.2760005593299866,
+      "logps/chosen": -578.0054931640625,
+      "logps/rejected": -620.122314453125,
+      "loss": 0.4064,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.5314385890960693,
+      "rewards/margins": 1.613104224205017,
+      "rewards/rejected": -4.144542694091797,
+      "step": 820
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.476818941645561e-09,
+      "logits/chosen": -0.822808563709259,
+      "logits/rejected": -0.3094715476036072,
+      "logps/chosen": -599.0675659179688,
+      "logps/rejected": -619.3136596679688,
+      "loss": 0.3862,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.7111198902130127,
+      "rewards/margins": 1.4173959493637085,
+      "rewards/rejected": -4.12851619720459,
+      "step": 830
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.609589412347347e-09,
+      "logits/chosen": -0.6289047002792358,
+      "logits/rejected": -0.11015300452709198,
+      "logps/chosen": -534.1783447265625,
+      "logps/rejected": -600.945556640625,
+      "loss": 0.3787,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -2.5174965858459473,
+      "rewards/margins": 1.8095569610595703,
+      "rewards/rejected": -4.327054023742676,
+      "step": 840
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.1280387858572667e-09,
+      "logits/chosen": -0.6604621410369873,
+      "logits/rejected": -0.242179274559021,
+      "logps/chosen": -580.7008056640625,
+      "logps/rejected": -598.622314453125,
+      "loss": 0.4136,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.638731002807617,
+      "rewards/margins": 1.3258345127105713,
+      "rewards/rejected": -3.9645659923553467,
+      "step": 850
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.03448615738172e-09,
+      "logits/chosen": -0.602516770362854,
+      "logits/rejected": -0.018761873245239258,
+      "logps/chosen": -582.8955688476562,
+      "logps/rejected": -606.2899169921875,
+      "loss": 0.4235,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.5163681507110596,
+      "rewards/margins": 1.8120934963226318,
+      "rewards/rejected": -4.328461647033691,
+      "step": 860
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.3064328257259575e-10,
+      "logits/chosen": -0.8884924054145813,
+      "logits/rejected": -0.2350286990404129,
+      "logps/chosen": -635.0806884765625,
+      "logps/rejected": -663.0999145507812,
+      "loss": 0.3724,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -2.5359692573547363,
+      "rewards/margins": 1.9048057794570923,
+      "rewards/rejected": -4.440774917602539,
+      "step": 870
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.7611898088715216e-11,
+      "logits/chosen": -0.7480968832969666,
+      "logits/rejected": -0.27411407232284546,
+      "logps/chosen": -547.1424560546875,
+      "logps/rejected": -604.2337646484375,
+      "loss": 0.361,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -2.3420045375823975,
+      "rewards/margins": 1.844956636428833,
+      "rewards/rejected": -4.1869611740112305,
+      "step": 880
+    },
     {
       "epoch": 1.0,
-      "step": 405,
+      "step": 883,
       "total_flos": 0.0,
-      "train_loss": 0.07294640629379838,
-      "train_runtime": 3765.6331,
-      "train_samples_per_second": 13.781,
-      "train_steps_per_second": 0.108
+      "train_loss": 0.43856339019935237,
+      "train_runtime": 7937.4578,
+      "train_samples_per_second": 14.24,
+      "train_steps_per_second": 0.111
     }
   ],
   "logging_steps": 10,
-  "max_steps": 405,
+  "max_steps": 883,
   "num_train_epochs": 1,
   "save_steps": 100,
   "total_flos": 0.0,