Model save

README.md

@@ -1,4 +1,5 @@
 ---
+base_model: princeton-nlp/Mistral-7B-Base-SFT-SimPO
 tags:
 - trl
 - dpo
@@ -13,7 +14,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model was trained from scratch on the None dataset.
+This model is a fine-tuned version of [princeton-nlp/Mistral-7B-Base-SFT-SimPO](https://huggingface.co/princeton-nlp/Mistral-7B-Base-SFT-SimPO) on the None dataset.
 
 ## Model description
 
@@ -32,7 +33,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 4e-07
+- learning_rate: 1e-07
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 3.8421780889894426,
-    "train_runtime": 6381.4933,
-    "train_samples": 52922,
-    "train_samples_per_second": 8.293,
-    "train_steps_per_second": 0.065
+    "train_loss": 0.5863300847029632,
+    "train_runtime": 6843.0852,
+    "train_samples": 56236,
+    "train_samples_per_second": 8.218,
+    "train_steps_per_second": 0.064
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4e88c6ac9b4127cd43c95069301c9c933118b1525797db81fc6d80f544cf7cec
+oid sha256:d085da4a857fb7f218891d9be471fb86e732b224ff99d833d19b15d462e4ce9d
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd64f1edcaa32df3d1b32cd35af92f119cd07ea2ba7ebf00b6828eed52da98f0
+oid sha256:7a5dbc0fe2ece97438d30fd6b0f0e3d1f3e734e6156b8e9d5f3c15a5dcaa7cf0
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87c91c34961d0019bf71d071d06cfd82742211fd4d3379b61e5bc9cad550096c
+oid sha256:220289bd03f2c16bc923749307f2d3eab7817266cb64a00415b0df5008ec88f8
 size 4540516344

runs/Jul01_23-19-09_n136-129-074/events.out.tfevents.1719847173.n136-129-074.2002672.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4497f7ce1c9d1f223465b26c89f67a43949fc300dc8929a9894ceb64c8f39e0a
-size 32977
+oid sha256:ca5bc1674e38eb08c326a1eb43f08b14e751071a2de1f6aeb0483afae3355ad5
+size 35395

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 3.8421780889894426,
-    "train_runtime": 6381.4933,
-    "train_samples": 52922,
-    "train_samples_per_second": 8.293,
-    "train_steps_per_second": 0.065
+    "train_loss": 0.5863300847029632,
+    "train_runtime": 6843.0852,
+    "train_samples": 56236,
+    "train_samples_per_second": 8.218,
+    "train_steps_per_second": 0.064
 }

trainer_state.json

@@ -1,22 +1,22 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9987908101571947,
+  "epoch": 0.9988623435722411,
   "eval_steps": 10000000,
-  "global_step": 413,
+  "global_step": 439,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 6401.270603874373,
-      "learning_rate": 9.523809523809522e-09,
-      "logits/chosen": -2.7005977630615234,
-      "logits/rejected": -2.6288318634033203,
-      "logps/chosen": -1.1158788204193115,
-      "logps/rejected": -1.1333446502685547,
-      "loss": 0.7544,
+      "grad_norm": 22.78893022336454,
+      "learning_rate": 2.2727272727272727e-09,
+      "logits/chosen": -1.6768856048583984,
+      "logits/rejected": -1.7259055376052856,
+      "logps/chosen": -1.2793102264404297,
+      "logps/rejected": -1.2162058353424072,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,631 +25,661 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 9369.590990783972,
-      "learning_rate": 9.523809523809523e-08,
-      "logits/chosen": -2.76228666305542,
-      "logits/rejected": -2.6970374584198,
-      "logps/chosen": -0.837486743927002,
-      "logps/rejected": -0.8182350993156433,
-      "loss": 0.9695,
-      "rewards/accuracies": 0.4305555522441864,
-      "rewards/chosen": 0.06597563624382019,
-      "rewards/margins": 0.437710702419281,
-      "rewards/rejected": -0.3717350959777832,
+      "grad_norm": 25.567365971420195,
+      "learning_rate": 2.2727272727272725e-08,
+      "logits/chosen": -1.7031302452087402,
+      "logits/rejected": -1.6688512563705444,
+      "logps/chosen": -1.213205337524414,
+      "logps/rejected": -1.220165729522705,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4513888955116272,
+      "rewards/chosen": 0.0002006387512665242,
+      "rewards/margins": -0.0009609226835891604,
+      "rewards/rejected": 0.001161561463959515,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 5966.657402243146,
-      "learning_rate": 1.9047619047619045e-07,
-      "logits/chosen": -2.6901049613952637,
-      "logits/rejected": -2.6502909660339355,
-      "logps/chosen": -0.9933319091796875,
-      "logps/rejected": -1.0394352674484253,
-      "loss": 1.0318,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.794396698474884,
-      "rewards/margins": 0.7471516728401184,
-      "rewards/rejected": -1.5415483713150024,
+      "grad_norm": 26.2968319036787,
+      "learning_rate": 4.545454545454545e-08,
+      "logits/chosen": -1.7800958156585693,
+      "logits/rejected": -1.7349421977996826,
+      "logps/chosen": -1.1448661088943481,
+      "logps/rejected": -1.185571312904358,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.0032871514558792114,
+      "rewards/margins": 0.0009361729025840759,
+      "rewards/rejected": -0.004223324358463287,
       "step": 20
     },
     {
       "epoch": 0.07,
-      "grad_norm": 8820.198504372876,
-      "learning_rate": 2.857142857142857e-07,
-      "logits/chosen": -2.7333264350891113,
-      "logits/rejected": -2.6793360710144043,
-      "logps/chosen": -0.9710652232170105,
-      "logps/rejected": -0.9799602627754211,
-      "loss": 1.3198,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -2.275942325592041,
-      "rewards/margins": 0.9020620584487915,
-      "rewards/rejected": -3.178004264831543,
+      "grad_norm": 28.64228481569523,
+      "learning_rate": 6.818181818181817e-08,
+      "logits/chosen": -1.744109869003296,
+      "logits/rejected": -1.6754045486450195,
+      "logps/chosen": -1.195277452468872,
+      "logps/rejected": -1.2481118440628052,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.02475227788090706,
+      "rewards/margins": 0.011208651587367058,
+      "rewards/rejected": -0.035960931330919266,
       "step": 30
     },
     {
-      "epoch": 0.1,
-      "grad_norm": 8453.783513094899,
-      "learning_rate": 3.809523809523809e-07,
-      "logits/chosen": -2.6771621704101562,
-      "logits/rejected": -2.6321842670440674,
-      "logps/chosen": -0.989823043346405,
-      "logps/rejected": -0.9216930270195007,
-      "loss": 2.0555,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.6586966514587402,
-      "rewards/margins": 5.100310325622559,
-      "rewards/rejected": -5.759006500244141,
+      "epoch": 0.09,
+      "grad_norm": 19.497764912088467,
+      "learning_rate": 9.09090909090909e-08,
+      "logits/chosen": -1.7310603857040405,
+      "logits/rejected": -1.6648972034454346,
+      "logps/chosen": -1.2513717412948608,
+      "logps/rejected": -1.3350750207901,
+      "loss": 0.6798,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.08884630352258682,
+      "rewards/margins": 0.06275957077741623,
+      "rewards/rejected": -0.15160587430000305,
       "step": 40
     },
     {
-      "epoch": 0.12,
-      "grad_norm": 4192.139232222726,
-      "learning_rate": 3.995412608484087e-07,
-      "logits/chosen": -2.743403911590576,
-      "logits/rejected": -2.6878693103790283,
-      "logps/chosen": -0.9671042561531067,
-      "logps/rejected": -0.917597770690918,
-      "loss": 2.6495,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.16885781288146973,
-      "rewards/margins": 5.186079978942871,
-      "rewards/rejected": -5.35493803024292,
+      "epoch": 0.11,
+      "grad_norm": 24.1777149048992,
+      "learning_rate": 9.994307990108962e-08,
+      "logits/chosen": -1.6923043727874756,
+      "logits/rejected": -1.6273339986801147,
+      "logps/chosen": -1.2986948490142822,
+      "logps/rejected": -1.356567144393921,
+      "loss": 0.6636,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.18121571838855743,
+      "rewards/margins": 0.0755590870976448,
+      "rewards/rejected": -0.25677481293678284,
       "step": 50
     },
     {
-      "epoch": 0.15,
-      "grad_norm": 5643.860863524967,
-      "learning_rate": 3.976812391485896e-07,
-      "logits/chosen": -2.7438769340515137,
-      "logits/rejected": -2.676765203475952,
-      "logps/chosen": -0.911353588104248,
-      "logps/rejected": -0.9122518301010132,
-      "loss": 3.8047,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": 2.4976494312286377,
-      "rewards/margins": 5.426072120666504,
-      "rewards/rejected": -2.928422212600708,
+      "epoch": 0.14,
+      "grad_norm": 28.137865307641825,
+      "learning_rate": 9.959570405988094e-08,
+      "logits/chosen": -1.7212276458740234,
+      "logits/rejected": -1.6404285430908203,
+      "logps/chosen": -1.2889435291290283,
+      "logps/rejected": -1.3794549703598022,
+      "loss": 0.6598,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.35969024896621704,
+      "rewards/margins": 0.07716657221317291,
+      "rewards/rejected": -0.4368568956851959,
       "step": 60
     },
     {
-      "epoch": 0.17,
-      "grad_norm": 4497.230754903385,
-      "learning_rate": 3.9440458281608213e-07,
-      "logits/chosen": -2.740940570831299,
-      "logits/rejected": -2.7162723541259766,
-      "logps/chosen": -0.9154363870620728,
-      "logps/rejected": -0.868497371673584,
-      "loss": 3.6432,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 5.681364059448242,
-      "rewards/margins": 8.000432968139648,
-      "rewards/rejected": -2.319068431854248,
+      "epoch": 0.16,
+      "grad_norm": 21.253003888204287,
+      "learning_rate": 9.893476820924666e-08,
+      "logits/chosen": -1.8007967472076416,
+      "logits/rejected": -1.7120532989501953,
+      "logps/chosen": -1.476588487625122,
+      "logps/rejected": -1.5963420867919922,
+      "loss": 0.6499,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.5274931192398071,
+      "rewards/margins": 0.14617758989334106,
+      "rewards/rejected": -0.6736707091331482,
       "step": 70
     },
     {
-      "epoch": 0.19,
-      "grad_norm": 9843.974275847575,
-      "learning_rate": 3.897347732134074e-07,
-      "logits/chosen": -2.679215908050537,
-      "logits/rejected": -2.625516891479492,
-      "logps/chosen": -0.9146322011947632,
-      "logps/rejected": -1.0181081295013428,
-      "loss": 5.767,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -10.08639907836914,
-      "rewards/margins": 6.582289695739746,
-      "rewards/rejected": -16.668689727783203,
+      "epoch": 0.18,
+      "grad_norm": 26.777703545743453,
+      "learning_rate": 9.796445099843647e-08,
+      "logits/chosen": -1.7857911586761475,
+      "logits/rejected": -1.6999902725219727,
+      "logps/chosen": -1.5350762605667114,
+      "logps/rejected": -1.6896966695785522,
+      "loss": 0.654,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.6562157869338989,
+      "rewards/margins": 0.18818075954914093,
+      "rewards/rejected": -0.8443965911865234,
       "step": 80
     },
     {
-      "epoch": 0.22,
-      "grad_norm": 4773.013380320505,
-      "learning_rate": 3.8370527539794614e-07,
-      "logits/chosen": -2.6771388053894043,
-      "logits/rejected": -2.6291418075561523,
-      "logps/chosen": -1.003847360610962,
-      "logps/rejected": -1.0297266244888306,
-      "loss": 4.6354,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 4.863407611846924,
-      "rewards/margins": 9.78220272064209,
-      "rewards/rejected": -4.918795585632324,
+      "epoch": 0.2,
+      "grad_norm": 25.00309177460346,
+      "learning_rate": 9.669088708527066e-08,
+      "logits/chosen": -1.7340404987335205,
+      "logits/rejected": -1.6651910543441772,
+      "logps/chosen": -1.6042931079864502,
+      "logps/rejected": -1.6743539571762085,
+      "loss": 0.6399,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.737191379070282,
+      "rewards/margins": 0.12603236734867096,
+      "rewards/rejected": -0.8632237315177917,
       "step": 90
     },
     {
-      "epoch": 0.24,
-      "grad_norm": 3074.8663144850243,
-      "learning_rate": 3.763592983027255e-07,
-      "logits/chosen": -2.705735683441162,
-      "logits/rejected": -2.6605448722839355,
-      "logps/chosen": -0.9163268804550171,
-      "logps/rejected": -0.9396775960922241,
-      "loss": 5.8585,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -12.477940559387207,
-      "rewards/margins": 7.702305793762207,
-      "rewards/rejected": -20.180248260498047,
+      "epoch": 0.23,
+      "grad_norm": 27.969594098965942,
+      "learning_rate": 9.512212835085849e-08,
+      "logits/chosen": -1.772202730178833,
+      "logits/rejected": -1.6827232837677002,
+      "logps/chosen": -1.5563361644744873,
+      "logps/rejected": -1.69924795627594,
+      "loss": 0.6231,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.7757617831230164,
+      "rewards/margins": 0.20040392875671387,
+      "rewards/rejected": -0.9761656522750854,
       "step": 100
     },
     {
-      "epoch": 0.27,
-      "grad_norm": 4099.610429119441,
-      "learning_rate": 3.6774948509008527e-07,
-      "logits/chosen": -2.714970111846924,
-      "logits/rejected": -2.6705470085144043,
-      "logps/chosen": -0.9598251581192017,
-      "logps/rejected": -0.9319995641708374,
-      "loss": 5.1529,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 9.19798755645752,
-      "rewards/margins": 10.779365539550781,
-      "rewards/rejected": -1.5813770294189453,
+      "epoch": 0.25,
+      "grad_norm": 25.83218161515289,
+      "learning_rate": 9.326809299301306e-08,
+      "logits/chosen": -1.771267294883728,
+      "logits/rejected": -1.6669820547103882,
+      "logps/chosen": -1.600318193435669,
+      "logps/rejected": -1.7721306085586548,
+      "loss": 0.6211,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.7993522882461548,
+      "rewards/margins": 0.24041876196861267,
+      "rewards/rejected": -1.0397710800170898,
       "step": 110
     },
     {
-      "epoch": 0.29,
-      "grad_norm": 4599.711217449366,
-      "learning_rate": 3.579375358972288e-07,
-      "logits/chosen": -2.678779125213623,
-      "logits/rejected": -2.6315762996673584,
-      "logps/chosen": -0.9081487655639648,
-      "logps/rejected": -1.0060938596725464,
-      "loss": 4.0915,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -13.663342475891113,
-      "rewards/margins": 9.755656242370605,
-      "rewards/rejected": -23.418994903564453,
+      "epoch": 0.27,
+      "grad_norm": 26.667996957961257,
+      "learning_rate": 9.114050282021158e-08,
+      "logits/chosen": -1.767559289932251,
+      "logits/rejected": -1.7058799266815186,
+      "logps/chosen": -1.565065622329712,
+      "logps/rejected": -1.7299690246582031,
+      "loss": 0.6144,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.8350059390068054,
+      "rewards/margins": 0.2350219041109085,
+      "rewards/rejected": -1.0700278282165527,
       "step": 120
     },
     {
-      "epoch": 0.31,
-      "grad_norm": 4010.334966061441,
-      "learning_rate": 3.4699376567716156e-07,
-      "logits/chosen": -2.7230353355407715,
-      "logits/rejected": -2.684389591217041,
-      "logps/chosen": -0.8652521967887878,
-      "logps/rejected": -0.8799147605895996,
-      "loss": 4.4027,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 2.19469952583313,
-      "rewards/margins": 15.263641357421875,
-      "rewards/rejected": -13.068939208984375,
+      "epoch": 0.3,
+      "grad_norm": 25.330536994335382,
+      "learning_rate": 8.875280914254802e-08,
+      "logits/chosen": -1.765298843383789,
+      "logits/rejected": -1.6749998331069946,
+      "logps/chosen": -1.7646329402923584,
+      "logps/rejected": -1.9669040441513062,
+      "loss": 0.5993,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.1436058282852173,
+      "rewards/margins": 0.34769219160079956,
+      "rewards/rejected": -1.491297960281372,
       "step": 130
     },
     {
-      "epoch": 0.34,
-      "grad_norm": 5239.11146834966,
-      "learning_rate": 3.349966003036421e-07,
-      "logits/chosen": -2.689558506011963,
-      "logits/rejected": -2.649766445159912,
-      "logps/chosen": -0.9352903366088867,
-      "logps/rejected": -0.9416161775588989,
-      "loss": 4.7953,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -4.734063148498535,
-      "rewards/margins": 8.841203689575195,
-      "rewards/rejected": -13.575268745422363,
+      "epoch": 0.32,
+      "grad_norm": 24.002049854639584,
+      "learning_rate": 8.612010772821971e-08,
+      "logits/chosen": -1.7989845275878906,
+      "logits/rejected": -1.7554515600204468,
+      "logps/chosen": -1.8291784524917603,
+      "logps/rejected": -1.9761606454849243,
+      "loss": 0.5991,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -1.2305986881256104,
+      "rewards/margins": 0.3437841534614563,
+      "rewards/rejected": -1.5743829011917114,
       "step": 140
     },
     {
-      "epoch": 0.36,
-      "grad_norm": 5394.35498681908,
-      "learning_rate": 3.220320145511884e-07,
-      "logits/chosen": -2.7070841789245605,
-      "logits/rejected": -2.647737979888916,
-      "logps/chosen": -0.9441506266593933,
-      "logps/rejected": -0.9885166883468628,
-      "loss": 4.2219,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 5.9402689933776855,
-      "rewards/margins": 12.97706413269043,
-      "rewards/rejected": -7.036795139312744,
+      "epoch": 0.34,
+      "grad_norm": 28.7685379938573,
+      "learning_rate": 8.325904336322055e-08,
+      "logits/chosen": -1.777856469154358,
+      "logits/rejected": -1.7238337993621826,
+      "logps/chosen": -1.9271425008773804,
+      "logps/rejected": -2.141960859298706,
+      "loss": 0.6082,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5195552110671997,
+      "rewards/margins": 0.3352898359298706,
+      "rewards/rejected": -1.8548450469970703,
       "step": 150
     },
     {
-      "epoch": 0.39,
-      "grad_norm": 5022.189692479379,
-      "learning_rate": 3.0819291597771795e-07,
-      "logits/chosen": -2.7051825523376465,
-      "logits/rejected": -2.667494297027588,
-      "logps/chosen": -0.911395251750946,
-      "logps/rejected": -0.939487099647522,
-      "loss": 4.7963,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -0.6114660501480103,
-      "rewards/margins": 9.443866729736328,
-      "rewards/rejected": -10.055331230163574,
+      "epoch": 0.36,
+      "grad_norm": 22.36425985091183,
+      "learning_rate": 8.01877046176447e-08,
+      "logits/chosen": -1.72470223903656,
+      "logits/rejected": -1.6554687023162842,
+      "logps/chosen": -1.9798545837402344,
+      "logps/rejected": -2.1781439781188965,
+      "loss": 0.5901,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -1.6031091213226318,
+      "rewards/margins": 0.29380664229393005,
+      "rewards/rejected": -1.8969157934188843,
       "step": 160
     },
     {
-      "epoch": 0.41,
-      "grad_norm": 5428.944545727042,
-      "learning_rate": 2.9357847912507786e-07,
-      "logits/chosen": -2.6787288188934326,
-      "logits/rejected": -2.609421968460083,
-      "logps/chosen": -0.8976411819458008,
-      "logps/rejected": -0.8857674598693848,
-      "loss": 4.6262,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.82297945022583,
-      "rewards/margins": 10.640687942504883,
-      "rewards/rejected": -13.463666915893555,
+      "epoch": 0.39,
+      "grad_norm": 19.908864367961982,
+      "learning_rate": 7.692550948392249e-08,
+      "logits/chosen": -1.7723356485366821,
+      "logits/rejected": -1.7198549509048462,
+      "logps/chosen": -1.9558120965957642,
+      "logps/rejected": -2.159667491912842,
+      "loss": 0.5921,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.4911130666732788,
+      "rewards/margins": 0.3868214190006256,
+      "rewards/rejected": -1.877934217453003,
       "step": 170
     },
     {
-      "epoch": 0.44,
-      "grad_norm": 7317.882582449178,
-      "learning_rate": 2.7829343480875617e-07,
-      "logits/chosen": -2.6716930866241455,
-      "logits/rejected": -2.6018152236938477,
-      "logps/chosen": -0.9342878460884094,
-      "logps/rejected": -0.9536906480789185,
-      "loss": 4.5209,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 4.173262596130371,
-      "rewards/margins": 8.933877944946289,
-      "rewards/rejected": -4.760615348815918,
+      "epoch": 0.41,
+      "grad_norm": 31.864529340507623,
+      "learning_rate": 7.349308261002021e-08,
+      "logits/chosen": -1.7342097759246826,
+      "logits/rejected": -1.6827507019042969,
+      "logps/chosen": -1.9941844940185547,
+      "logps/rejected": -2.22194504737854,
+      "loss": 0.5855,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -1.5941965579986572,
+      "rewards/margins": 0.35622045397758484,
+      "rewards/rejected": -1.950416922569275,
       "step": 180
     },
     {
-      "epoch": 0.46,
-      "grad_norm": 5046.2946182405685,
-      "learning_rate": 2.624473195899052e-07,
-      "logits/chosen": -2.737992763519287,
-      "logits/rejected": -2.7089955806732178,
-      "logps/chosen": -0.9629039764404297,
-      "logps/rejected": -1.039236307144165,
-      "loss": 4.5521,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -3.3569388389587402,
-      "rewards/margins": 13.995905876159668,
-      "rewards/rejected": -17.352848052978516,
+      "epoch": 0.43,
+      "grad_norm": 24.691918932601595,
+      "learning_rate": 6.991212490377531e-08,
+      "logits/chosen": -1.7905107736587524,
+      "logits/rejected": -1.7474826574325562,
+      "logps/chosen": -2.10019850730896,
+      "logps/rejected": -2.3696181774139404,
+      "loss": 0.5604,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.740191102027893,
+      "rewards/margins": 0.4957551956176758,
+      "rewards/rejected": -2.2359461784362793,
       "step": 190
     },
     {
-      "epoch": 0.48,
-      "grad_norm": 4268.8163809344915,
-      "learning_rate": 2.4615369080815547e-07,
-      "logits/chosen": -2.6982626914978027,
-      "logits/rejected": -2.6629488468170166,
-      "logps/chosen": -0.8523995280265808,
-      "logps/rejected": -0.9246847033500671,
-      "loss": 3.8184,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 2.3979854583740234,
-      "rewards/margins": 4.996596336364746,
-      "rewards/rejected": -2.5986106395721436,
+      "epoch": 0.46,
+      "grad_norm": 30.11763021258028,
+      "learning_rate": 6.620527633276978e-08,
+      "logits/chosen": -1.7320966720581055,
+      "logits/rejected": -1.6665375232696533,
+      "logps/chosen": -2.161935329437256,
+      "logps/rejected": -2.522731304168701,
+      "loss": 0.5808,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.9349641799926758,
+      "rewards/margins": 0.5480148792266846,
+      "rewards/rejected": -2.4829792976379395,
       "step": 200
     },
     {
-      "epoch": 0.51,
-      "grad_norm": 4211.242306423206,
-      "learning_rate": 2.2952931280049625e-07,
-      "logits/chosen": -2.7346115112304688,
-      "logits/rejected": -2.6734609603881836,
-      "logps/chosen": -1.0063531398773193,
-      "logps/rejected": -0.9570119976997375,
-      "loss": 4.9954,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 7.86081600189209,
-      "rewards/margins": 13.075413703918457,
-      "rewards/rejected": -5.214597225189209,
+      "epoch": 0.48,
+      "grad_norm": 27.459036499855436,
+      "learning_rate": 6.239597278716581e-08,
+      "logits/chosen": -1.7859094142913818,
+      "logits/rejected": -1.7306747436523438,
+      "logps/chosen": -2.2954204082489014,
+      "logps/rejected": -2.540391445159912,
+      "loss": 0.5619,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.1001808643341064,
+      "rewards/margins": 0.5312689542770386,
+      "rewards/rejected": -2.6314499378204346,
       "step": 210
     },
     {
-      "epoch": 0.53,
-      "grad_norm": 4906.2448320907815,
-      "learning_rate": 2.1269332013798747e-07,
-      "logits/chosen": -2.7431142330169678,
-      "logits/rejected": -2.7241249084472656,
-      "logps/chosen": -0.8835189938545227,
-      "logps/rejected": -0.8670462369918823,
-      "loss": 4.3795,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 4.582579135894775,
-      "rewards/margins": 8.150335311889648,
-      "rewards/rejected": -3.567755937576294,
+      "epoch": 0.5,
+      "grad_norm": 26.562029580561404,
+      "learning_rate": 5.8508297910462456e-08,
+      "logits/chosen": -1.7313369512557983,
+      "logits/rejected": -1.6572465896606445,
+      "logps/chosen": -2.212656021118164,
+      "logps/rejected": -2.5422561168670654,
+      "loss": 0.5673,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.0982813835144043,
+      "rewards/margins": 0.5436533689498901,
+      "rewards/rejected": -2.641934871673584,
       "step": 220
     },
     {
-      "epoch": 0.56,
-      "grad_norm": 2995.5119741253625,
-      "learning_rate": 1.9576636387676436e-07,
-      "logits/chosen": -2.690732955932617,
-      "logits/rejected": -2.653067111968994,
-      "logps/chosen": -0.960831344127655,
-      "logps/rejected": -0.9556485414505005,
-      "loss": 4.0487,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -8.449748039245605,
-      "rewards/margins": 10.095115661621094,
-      "rewards/rejected": -18.544864654541016,
+      "epoch": 0.52,
+      "grad_norm": 26.043734197401424,
+      "learning_rate": 5.456683083494731e-08,
+      "logits/chosen": -1.7218725681304932,
+      "logits/rejected": -1.6862335205078125,
+      "logps/chosen": -2.1797163486480713,
+      "logps/rejected": -2.389193058013916,
+      "loss": 0.5843,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.9718148708343506,
+      "rewards/margins": 0.37147068977355957,
+      "rewards/rejected": -2.34328556060791,
       "step": 230
     },
     {
-      "epoch": 0.58,
-      "grad_norm": 3504.414329050279,
-      "learning_rate": 1.7886974694151976e-07,
-      "logits/chosen": -2.7119805812835693,
-      "logits/rejected": -2.6879172325134277,
-      "logps/chosen": -0.990290641784668,
-      "logps/rejected": -0.9934972524642944,
-      "loss": 4.3644,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.5965616106987,
-      "rewards/margins": 9.211602210998535,
-      "rewards/rejected": -9.808164596557617,
+      "epoch": 0.55,
+      "grad_norm": 33.60349201324705,
+      "learning_rate": 5.059649078450834e-08,
+      "logits/chosen": -1.7049477100372314,
+      "logits/rejected": -1.6663001775741577,
+      "logps/chosen": -2.2287259101867676,
+      "logps/rejected": -2.5061872005462646,
+      "loss": 0.5527,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -2.120744466781616,
+      "rewards/margins": 0.4308454990386963,
+      "rewards/rejected": -2.5515899658203125,
       "step": 240
     },
     {
-      "epoch": 0.6,
-      "grad_norm": 3400.5848210057216,
-      "learning_rate": 1.6212455483752895e-07,
-      "logits/chosen": -2.756906270980835,
-      "logits/rejected": -2.6796135902404785,
-      "logps/chosen": -0.8838168978691101,
-      "logps/rejected": -0.9137406349182129,
-      "loss": 4.5034,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 6.5281982421875,
-      "rewards/margins": 8.64702033996582,
-      "rewards/rejected": -2.1188230514526367,
+      "epoch": 0.57,
+      "grad_norm": 24.721682812448297,
+      "learning_rate": 4.6622379527277186e-08,
+      "logits/chosen": -1.716957688331604,
+      "logits/rejected": -1.6692520380020142,
+      "logps/chosen": -2.323068857192993,
+      "logps/rejected": -2.551687240600586,
+      "loss": 0.5645,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.3683600425720215,
+      "rewards/margins": 0.3910773694515228,
+      "rewards/rejected": -2.759437322616577,
       "step": 250
     },
     {
-      "epoch": 0.63,
-      "grad_norm": 6194.117841583386,
-      "learning_rate": 1.4565078792075733e-07,
-      "logits/chosen": -2.7132773399353027,
-      "logits/rejected": -2.6494650840759277,
-      "logps/chosen": -1.002362847328186,
-      "logps/rejected": -0.9982520341873169,
-      "loss": 4.8134,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 3.0224878787994385,
-      "rewards/margins": 16.206506729125977,
-      "rewards/rejected": -13.1840181350708,
+      "epoch": 0.59,
+      "grad_norm": 28.965518895994943,
+      "learning_rate": 4.26696226741691e-08,
+      "logits/chosen": -1.731431007385254,
+      "logits/rejected": -1.6687753200531006,
+      "logps/chosen": -2.4228413105010986,
+      "logps/rejected": -2.701592206954956,
+      "loss": 0.5647,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.380035877227783,
+      "rewards/margins": 0.5175679922103882,
+      "rewards/rejected": -2.897603750228882,
       "step": 260
     },
     {
-      "epoch": 0.65,
-      "grad_norm": 4565.495892627232,
-      "learning_rate": 1.295665014444281e-07,
-      "logits/chosen": -2.7381529808044434,
-      "logits/rejected": -2.6608738899230957,
-      "logps/chosen": -0.9501218795776367,
-      "logps/rejected": -0.9476363062858582,
-      "loss": 5.3754,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -0.570526123046875,
-      "rewards/margins": 12.367398262023926,
-      "rewards/rejected": -12.9379243850708,
+      "epoch": 0.61,
+      "grad_norm": 24.276560436452733,
+      "learning_rate": 3.876321082668098e-08,
+      "logits/chosen": -1.7877088785171509,
+      "logits/rejected": -1.7258117198944092,
+      "logps/chosen": -2.3104138374328613,
+      "logps/rejected": -2.597568988800049,
+      "loss": 0.5577,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.143846035003662,
+      "rewards/margins": 0.5556932091712952,
+      "rewards/rejected": -2.6995394229888916,
       "step": 270
     },
     {
-      "epoch": 0.68,
-      "grad_norm": 5337.153187944306,
-      "learning_rate": 1.1398695954469597e-07,
-      "logits/chosen": -2.6872425079345703,
-      "logits/rejected": -2.630267381668091,
-      "logps/chosen": -0.9056104421615601,
-      "logps/rejected": -0.8939152956008911,
-      "loss": 4.1053,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 4.294297218322754,
-      "rewards/margins": 7.472552299499512,
-      "rewards/rejected": -3.1782548427581787,
+      "epoch": 0.64,
+      "grad_norm": 26.01398581499373,
+      "learning_rate": 3.492784157826244e-08,
+      "logits/chosen": -1.7255363464355469,
+      "logits/rejected": -1.6368858814239502,
+      "logps/chosen": -2.307375431060791,
+      "logps/rejected": -2.6467177867889404,
+      "loss": 0.5569,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.1689984798431396,
+      "rewards/margins": 0.6079045534133911,
+      "rewards/rejected": -2.7769031524658203,
       "step": 280
     },
     {
-      "epoch": 0.7,
-      "grad_norm": 3582.07962645892,
-      "learning_rate": 9.902380922818425e-08,
-      "logits/chosen": -2.7334370613098145,
-      "logits/rejected": -2.6919913291931152,
-      "logps/chosen": -0.9840775728225708,
-      "logps/rejected": -0.9756690263748169,
-      "loss": 3.2759,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 8.966680526733398,
-      "rewards/margins": 11.496904373168945,
-      "rewards/rejected": -2.5302233695983887,
+      "epoch": 0.66,
+      "grad_norm": 22.449000475495634,
+      "learning_rate": 3.118776336817812e-08,
+      "logits/chosen": -1.7589473724365234,
+      "logits/rejected": -1.7079540491104126,
+      "logps/chosen": -2.2426087856292725,
+      "logps/rejected": -2.575594425201416,
+      "loss": 0.5461,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.0787200927734375,
+      "rewards/margins": 0.6515394449234009,
+      "rewards/rejected": -2.730259656906128,
       "step": 290
     },
     {
-      "epoch": 0.73,
-      "grad_norm": 4767.591882910886,
-      "learning_rate": 8.478428028080398e-08,
-      "logits/chosen": -2.7305169105529785,
-      "logits/rejected": -2.6773815155029297,
-      "logps/chosen": -0.8988749384880066,
-      "logps/rejected": -0.9437707662582397,
-      "loss": 4.3175,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.8898951411247253,
-      "rewards/margins": 8.447718620300293,
-      "rewards/rejected": -7.55782413482666,
+      "epoch": 0.68,
+      "grad_norm": 23.916342153040954,
+      "learning_rate": 2.7566622175067443e-08,
+      "logits/chosen": -1.7405074834823608,
+      "logits/rejected": -1.6836473941802979,
+      "logps/chosen": -2.341914176940918,
+      "logps/rejected": -2.724348545074463,
+      "loss": 0.5507,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.2666609287261963,
+      "rewards/margins": 0.6088961958885193,
+      "rewards/rejected": -2.875556707382202,
       "step": 300
     },
     {
-      "epoch": 0.75,
-      "grad_norm": 4819.380329592898,
-      "learning_rate": 7.137041683151202e-08,
-      "logits/chosen": -2.7228643894195557,
-      "logits/rejected": -2.6581058502197266,
-      "logps/chosen": -1.0781683921813965,
-      "logps/rejected": -1.028840184211731,
-      "loss": 2.9744,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -2.4226202964782715,
-      "rewards/margins": 13.473236083984375,
-      "rewards/rejected": -15.895855903625488,
+      "epoch": 0.71,
+      "grad_norm": 28.87016818276154,
+      "learning_rate": 2.408731201945432e-08,
+      "logits/chosen": -1.738867998123169,
+      "logits/rejected": -1.6946017742156982,
+      "logps/chosen": -2.2607665061950684,
+      "logps/rejected": -2.508131742477417,
+      "loss": 0.5549,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -2.133742332458496,
+      "rewards/margins": 0.45442089438438416,
+      "rewards/rejected": -2.5881636142730713,
       "step": 310
     },
     {
-      "epoch": 0.77,
-      "grad_norm": 7840.551721640683,
-      "learning_rate": 5.8878346077822135e-08,
-      "logits/chosen": -2.7280871868133545,
-      "logits/rejected": -2.649958848953247,
-      "logps/chosen": -0.9020591974258423,
-      "logps/rejected": -0.9361578822135925,
-      "loss": 2.7082,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 1.223115086555481,
-      "rewards/margins": 10.582406997680664,
-      "rewards/rejected": -9.35929012298584,
+      "epoch": 0.73,
+      "grad_norm": 26.474891102018546,
+      "learning_rate": 2.0771830220378112e-08,
+      "logits/chosen": -1.6925548315048218,
+      "logits/rejected": -1.6349446773529053,
+      "logps/chosen": -2.292504072189331,
+      "logps/rejected": -2.5281872749328613,
+      "loss": 0.5545,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.139239549636841,
+      "rewards/margins": 0.48706698417663574,
+      "rewards/rejected": -2.6263065338134766,
       "step": 320
     },
     {
-      "epoch": 0.8,
-      "grad_norm": 4662.77535052248,
-      "learning_rate": 4.73975894135696e-08,
-      "logits/chosen": -2.6770853996276855,
-      "logits/rejected": -2.6099040508270264,
-      "logps/chosen": -0.9263202548027039,
-      "logps/rejected": -0.9608638882637024,
-      "loss": 3.1985,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.2365754395723343,
-      "rewards/margins": 13.195585250854492,
-      "rewards/rejected": -12.959010124206543,
+      "epoch": 0.75,
+      "grad_norm": 26.153240463475917,
+      "learning_rate": 1.7641138321260257e-08,
+      "logits/chosen": -1.7273342609405518,
+      "logits/rejected": -1.6638519763946533,
+      "logps/chosen": -2.218357563018799,
+      "logps/rejected": -2.6145200729370117,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.069005250930786,
+      "rewards/margins": 0.7372555136680603,
+      "rewards/rejected": -2.806260585784912,
       "step": 330
     },
     {
-      "epoch": 0.82,
-      "grad_norm": 4550.588002339864,
-      "learning_rate": 3.701042089556483e-08,
-      "logits/chosen": -2.756493330001831,
-      "logits/rejected": -2.687851667404175,
-      "logps/chosen": -0.8901381492614746,
-      "logps/rejected": -0.9301478266716003,
-      "loss": 3.841,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 0.11963929980993271,
-      "rewards/margins": 7.3289618492126465,
-      "rewards/rejected": -7.209322929382324,
+      "epoch": 0.77,
+      "grad_norm": 26.27728864294511,
+      "learning_rate": 1.4715029564277793e-08,
+      "logits/chosen": -1.7901878356933594,
+      "logits/rejected": -1.7485193014144897,
+      "logps/chosen": -2.2015347480773926,
+      "logps/rejected": -2.5572023391723633,
+      "loss": 0.5577,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -1.9937489032745361,
+      "rewards/margins": 0.637208104133606,
+      "rewards/rejected": -2.6309566497802734,
       "step": 340
     },
     {
-      "epoch": 0.85,
-      "grad_norm": 5464.471487236709,
-      "learning_rate": 2.779127764652889e-08,
-      "logits/chosen": -2.689107656478882,
-      "logits/rejected": -2.6330015659332275,
-      "logps/chosen": -0.9756801724433899,
-      "logps/rejected": -0.9646003842353821,
-      "loss": 3.6421,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.1974527835845947,
-      "rewards/margins": 9.013090133666992,
-      "rewards/rejected": -10.210542678833008,
+      "epoch": 0.8,
+      "grad_norm": 26.035613152091976,
+      "learning_rate": 1.2012003751113343e-08,
+      "logits/chosen": -1.7805286645889282,
+      "logits/rejected": -1.7256838083267212,
+      "logps/chosen": -2.3138480186462402,
+      "logps/rejected": -2.6945948600769043,
+      "loss": 0.536,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.3033559322357178,
+      "rewards/margins": 0.6611131429672241,
+      "rewards/rejected": -2.9644687175750732,
       "step": 350
     },
     {
-      "epoch": 0.87,
-      "grad_norm": 5949.708940984834,
-      "learning_rate": 1.9806226419516193e-08,
-      "logits/chosen": -2.704460620880127,
-      "logits/rejected": -2.656071186065674,
-      "logps/chosen": -0.9623576402664185,
-      "logps/rejected": -1.0082406997680664,
-      "loss": 3.5231,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 2.5273587703704834,
-      "rewards/margins": 11.88086223602295,
-      "rewards/rejected": -9.35350227355957,
+      "epoch": 0.82,
+      "grad_norm": 31.4301067051425,
+      "learning_rate": 9.549150281252633e-09,
+      "logits/chosen": -1.7304834127426147,
+      "logits/rejected": -1.6880794763565063,
+      "logps/chosen": -2.2502734661102295,
+      "logps/rejected": -2.593324661254883,
+      "loss": 0.54,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.1562764644622803,
+      "rewards/margins": 0.5826417207717896,
+      "rewards/rejected": -2.7389183044433594,
       "step": 360
     },
     {
-      "epoch": 0.89,
-      "grad_norm": 4320.933402478669,
-      "learning_rate": 1.3112490146559552e-08,
-      "logits/chosen": -2.7451281547546387,
-      "logits/rejected": -2.686728000640869,
-      "logps/chosen": -0.8951610326766968,
-      "logps/rejected": -0.89850914478302,
-      "loss": 3.0053,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 0.6753175258636475,
-      "rewards/margins": 12.29626750946045,
-      "rewards/rejected": -11.620949745178223,
+      "epoch": 0.84,
+      "grad_norm": 25.13509986712804,
+      "learning_rate": 7.3420401072985306e-09,
+      "logits/chosen": -1.7723455429077148,
+      "logits/rejected": -1.721980094909668,
+      "logps/chosen": -2.3107597827911377,
+      "logps/rejected": -2.679028272628784,
+      "loss": 0.5374,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.1922922134399414,
+      "rewards/margins": 0.6216103434562683,
+      "rewards/rejected": -2.8139023780822754,
       "step": 370
     },
     {
-      "epoch": 0.92,
-      "grad_norm": 2514.940389992379,
-      "learning_rate": 7.758037864413247e-09,
-      "logits/chosen": -2.7158432006835938,
-      "logits/rejected": -2.6906635761260986,
-      "logps/chosen": -0.9033122062683105,
-      "logps/rejected": -0.9709407091140747,
-      "loss": 2.8751,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 0.9628832936286926,
-      "rewards/margins": 10.83133316040039,
-      "rewards/rejected": -9.868449211120605,
+      "epoch": 0.86,
+      "grad_norm": 28.235691242335875,
+      "learning_rate": 5.404627290395369e-09,
+      "logits/chosen": -1.733109712600708,
+      "logits/rejected": -1.6734564304351807,
+      "logps/chosen": -2.241391897201538,
+      "logps/rejected": -2.590752363204956,
+      "loss": 0.5365,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -2.103571653366089,
+      "rewards/margins": 0.6187294721603394,
+      "rewards/rejected": -2.7223010063171387,
       "step": 380
     },
     {
-      "epoch": 0.94,
-      "grad_norm": 3504.225752431698,
-      "learning_rate": 3.78124095609087e-09,
-      "logits/chosen": -2.6947999000549316,
-      "logits/rejected": -2.6553878784179688,
-      "logps/chosen": -0.9263744354248047,
-      "logps/rejected": -0.9935058355331421,
-      "loss": 3.019,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 1.4072116613388062,
-      "rewards/margins": 10.741894721984863,
-      "rewards/rejected": -9.334683418273926,
+      "epoch": 0.89,
+      "grad_norm": 33.12956908569685,
+      "learning_rate": 3.74916077816162e-09,
+      "logits/chosen": -1.7393004894256592,
+      "logits/rejected": -1.6843922138214111,
+      "logps/chosen": -2.267721652984619,
+      "logps/rejected": -2.5625317096710205,
+      "loss": 0.5513,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -2.235853672027588,
+      "rewards/margins": 0.5573619604110718,
+      "rewards/rejected": -2.7932159900665283,
       "step": 390
     },
     {
-      "epoch": 0.97,
-      "grad_norm": 4987.634749508018,
-      "learning_rate": 1.2105981716597603e-09,
-      "logits/chosen": -2.7300946712493896,
-      "logits/rejected": -2.6389007568359375,
-      "logps/chosen": -0.9686774015426636,
-      "logps/rejected": -0.9328421354293823,
-      "loss": 3.7864,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 0.6465551853179932,
-      "rewards/margins": 11.809611320495605,
-      "rewards/rejected": -11.163057327270508,
+      "epoch": 0.91,
+      "grad_norm": 28.33250387056753,
+      "learning_rate": 2.386106962899165e-09,
+      "logits/chosen": -1.666548490524292,
+      "logits/rejected": -1.6048628091812134,
+      "logps/chosen": -2.401040554046631,
+      "logps/rejected": -2.733664035797119,
+      "loss": 0.5422,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.349238872528076,
+      "rewards/margins": 0.5525677800178528,
+      "rewards/rejected": -2.901806592941284,
       "step": 400
     },
     {
-      "epoch": 0.99,
-      "grad_norm": 5473.226219590305,
-      "learning_rate": 6.453139886395398e-11,
-      "logits/chosen": -2.7284317016601562,
-      "logits/rejected": -2.6886637210845947,
-      "logps/chosen": -0.9334842562675476,
-      "logps/rejected": -0.9600637555122375,
-      "loss": 3.6391,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -2.157397747039795,
-      "rewards/margins": 13.2835054397583,
-      "rewards/rejected": -15.440902709960938,
+      "epoch": 0.93,
+      "grad_norm": 31.32317842504756,
+      "learning_rate": 1.3240835096913706e-09,
+      "logits/chosen": -1.6938998699188232,
+      "logits/rejected": -1.6031659841537476,
+      "logps/chosen": -2.251462459564209,
+      "logps/rejected": -2.6835711002349854,
+      "loss": 0.5597,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.1598236560821533,
+      "rewards/margins": 0.7253878116607666,
+      "rewards/rejected": -2.885211706161499,
       "step": 410
     },
+    {
+      "epoch": 0.96,
+      "grad_norm": 23.813879321456167,
+      "learning_rate": 5.698048727497462e-10,
+      "logits/chosen": -1.7287553548812866,
+      "logits/rejected": -1.6634016036987305,
+      "logps/chosen": -2.358034610748291,
+      "logps/rejected": -2.7596287727355957,
+      "loss": 0.5431,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -2.3334739208221436,
+      "rewards/margins": 0.669273853302002,
+      "rewards/rejected": -3.0027480125427246,
+      "step": 420
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 24.13495374841747,
+      "learning_rate": 1.2803984447259387e-10,
+      "logits/chosen": -1.7348169088363647,
+      "logits/rejected": -1.6923096179962158,
+      "logps/chosen": -2.339822769165039,
+      "logps/rejected": -2.7242085933685303,
+      "loss": 0.5338,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -2.314966917037964,
+      "rewards/margins": 0.6734664440155029,
+      "rewards/rejected": -2.9884331226348877,
+      "step": 430
+    },
     {
       "epoch": 1.0,
-      "step": 413,
+      "step": 439,
       "total_flos": 0.0,
-      "train_loss": 3.8421780889894426,
-      "train_runtime": 6381.4933,
-      "train_samples_per_second": 8.293,
-      "train_steps_per_second": 0.065
+      "train_loss": 0.5863300847029632,
+      "train_runtime": 6843.0852,
+      "train_samples_per_second": 8.218,
+      "train_steps_per_second": 0.064
     }
   ],
   "logging_steps": 10,
-  "max_steps": 413,
+  "max_steps": 439,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,