Model save

README.md

@@ -17,15 +17,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.7774
-- Rewards/chosen: -2.3054
-- Rewards/rejected: -3.6762
-- Rewards/accuracies: 0.7857
-- Rewards/margins: 1.3708
-- Logps/rejected: -629.3941
-- Logps/chosen: -514.6497
-- Logits/rejected: 3.0974
-- Logits/chosen: 1.8746
+- Loss: 0.5046
+- Rewards/chosen: -1.1826
+- Rewards/rejected: -2.0581
+- Rewards/accuracies: 0.7246
+- Rewards/margins: 0.8756
+- Logps/rejected: -470.5493
+- Logps/chosen: -395.9858
+- Logits/rejected: 0.0457
+- Logits/chosen: -0.4473
 
 ## Model description
 
@@ -45,14 +45,14 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-07
-- train_batch_size: 8
-- eval_batch_size: 8
+- train_batch_size: 4
+- eval_batch_size: 4
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 4
-- gradient_accumulation_steps: 2
-- total_train_batch_size: 64
-- total_eval_batch_size: 32
+- num_devices: 3
+- gradient_accumulation_steps: 10
+- total_train_batch_size: 120
+- total_eval_batch_size: 12
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
@@ -62,15 +62,11 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 1.0407        | 0.1   | 100  | 1.0319          | -0.3679        | -0.5944          | 0.6845             | 0.2265          | -321.2206      | -320.9028    | -2.4869         | -2.5233       |
-| 0.8874        | 0.21  | 200  | 0.8947          | -0.8206        | -1.5352          | 0.7480             | 0.7146          | -415.2939      | -366.1670    | -0.4438         | -0.9008       |
-| 0.8068        | 0.31  | 300  | 0.8382          | -1.4666        | -2.4781          | 0.7540             | 1.0115          | -509.5933      | -430.7722    | 1.4832          | 0.8117        |
-| 0.7845        | 0.42  | 400  | 0.8209          | -1.7788        | -2.9187          | 0.7520             | 1.1399          | -553.6510      | -461.9887    | 2.8755          | 2.1264        |
-| 0.8323        | 0.52  | 500  | 0.8332          | -1.4352        | -2.5462          | 0.7440             | 1.1110          | -516.3953      | -427.6284    | 2.1308          | 1.3306        |
-| 0.7677        | 0.63  | 600  | 0.7981          | -2.1915        | -3.5501          | 0.7520             | 1.3586          | -616.7921      | -503.2610    | 3.3207          | 1.8966        |
-| 0.7227        | 0.73  | 700  | 0.7834          | -2.2316        | -3.6191          | 0.7639             | 1.3876          | -623.6929      | -507.2672    | 2.9762          | 1.7450        |
-| 0.7455        | 0.84  | 800  | 0.7792          | -2.4217        | -3.8218          | 0.7758             | 1.4001          | -643.9626      | -526.2844    | 3.2439          | 1.9906        |
-| 0.7785        | 0.94  | 900  | 0.7779          | -2.2974        | -3.6650          | 0.7798             | 1.3675          | -628.2753      | -513.8542    | 3.0941          | 1.8722        |
+| 0.5764        | 0.2   | 100  | 0.5829          | -0.3592        | -0.7613          | 0.6931             | 0.4020          | -340.8605      | -313.6503    | -2.4360         | -2.4791       |
+| 0.5169        | 0.39  | 200  | 0.5312          | -0.8847        | -1.6204          | 0.7066             | 0.7356          | -426.7720      | -366.2012    | -0.8443         | -1.2010       |
+| 0.5133        | 0.59  | 300  | 0.5159          | -1.1886        | -1.9604          | 0.7246             | 0.7718          | -460.7765      | -396.5906    | 0.0460          | -0.3853       |
+| 0.4968        | 0.79  | 400  | 0.5058          | -1.2445        | -2.1063          | 0.7141             | 0.8618          | -475.3639      | -402.1766    | 0.2014          | -0.2552       |
+| 0.4833        | 0.98  | 500  | 0.5045          | -1.1821        | -2.0581          | 0.7260             | 0.8760          | -470.5448      | -395.9374    | 0.0436          | -0.4496       |
 
 
 ### Framework versions

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 1.8746349811553955,
-    "eval_logits/rejected": 3.097362756729126,
-    "eval_logps/chosen": -514.6497192382812,
-    "eval_logps/rejected": -629.3941040039062,
-    "eval_loss": 0.7774083614349365,
-    "eval_rewards/accuracies": 0.7857142686843872,
-    "eval_rewards/chosen": -2.305399179458618,
-    "eval_rewards/margins": 1.3707566261291504,
-    "eval_rewards/rejected": -3.6761555671691895,
-    "eval_runtime": 244.4926,
+    "eval_logits/chosen": -0.44726553559303284,
+    "eval_logits/rejected": 0.045745834708213806,
+    "eval_logps/chosen": -395.98583984375,
+    "eval_logps/rejected": -470.54931640625,
+    "eval_loss": 0.5046471357345581,
+    "eval_rewards/accuracies": 0.7245509028434753,
+    "eval_rewards/chosen": -1.1825801134109497,
+    "eval_rewards/margins": 0.8755642771720886,
+    "eval_rewards/rejected": -2.0581440925598145,
+    "eval_runtime": 494.7185,
     "eval_samples": 2000,
-    "eval_samples_per_second": 8.18,
-    "eval_steps_per_second": 0.258,
-    "train_loss": 0.8416462149295507,
-    "train_runtime": 20662.0179,
+    "eval_samples_per_second": 4.043,
+    "eval_steps_per_second": 0.338,
+    "train_loss": 0.5401819272219315,
+    "train_runtime": 34352.758,
     "train_samples": 61135,
-    "train_samples_per_second": 2.959,
-    "train_steps_per_second": 0.046
+    "train_samples_per_second": 1.78,
+    "train_steps_per_second": 0.015
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 1.8746349811553955,
-    "eval_logits/rejected": 3.097362756729126,
-    "eval_logps/chosen": -514.6497192382812,
-    "eval_logps/rejected": -629.3941040039062,
-    "eval_loss": 0.7774083614349365,
-    "eval_rewards/accuracies": 0.7857142686843872,
-    "eval_rewards/chosen": -2.305399179458618,
-    "eval_rewards/margins": 1.3707566261291504,
-    "eval_rewards/rejected": -3.6761555671691895,
-    "eval_runtime": 244.4926,
+    "eval_logits/chosen": -0.44726553559303284,
+    "eval_logits/rejected": 0.045745834708213806,
+    "eval_logps/chosen": -395.98583984375,
+    "eval_logps/rejected": -470.54931640625,
+    "eval_loss": 0.5046471357345581,
+    "eval_rewards/accuracies": 0.7245509028434753,
+    "eval_rewards/chosen": -1.1825801134109497,
+    "eval_rewards/margins": 0.8755642771720886,
+    "eval_rewards/rejected": -2.0581440925598145,
+    "eval_runtime": 494.7185,
     "eval_samples": 2000,
-    "eval_samples_per_second": 8.18,
-    "eval_steps_per_second": 0.258
+    "eval_samples_per_second": 4.043,
+    "eval_steps_per_second": 0.338
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25fd3bf4215bbd131ceb23b460f9d81e046f3c7f8036538a66eec9cf5df4d133
+oid sha256:bddcb477d44b03d21f482dfee881e87dc11347525cffd01a54e6bbe7b24b6083
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16fa4e673a137503a5ffd3f374fdfbeeaa6937c1a117948d3eb7a4b379896b40
+oid sha256:e5751b4a0c756ee1e97af6b65bce15ff65004fd334bd8f9068769e14fc685e3c
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88d174023efd91b4ab079ed43a18c1fc247282d94b9776c3e5cd126b9c4cef87
+oid sha256:e9f5e849324cd1452cd76c40ab2420e8932e50bd8cdf6876faad637044d35237
 size 4540516344

runs/Oct26_07-09-09_RLHF000/events.out.tfevents.1729926715.RLHF000.205082.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8198256bf241db9ffcc9aaa0a30c95c52049316afc0c6cd7d6050d3ef7a723a9
+size 4505

runs/Oct26_07-40-14_RLHF000/events.out.tfevents.1729928440.RLHF000.213828.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5a4bcd2eb605dbeb99a9139d4de439c91723d61ae9cbe5cccc5bf5943a3310c
+size 5128

runs/Oct26_07-43-10_RLHF000/events.out.tfevents.1729928610.RLHF000.215421.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:372271a3da8d1fdb5d70213071193f32bbcf69751d7b5e75e6fcc0c0448e58b2
+size 4506

runs/Oct26_07-44-28_RLHF000/events.out.tfevents.1729928694.RLHF000.216128.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:162190b88bd4e01f86146512d2834e576a7ae2ad1464214e871175d6b9e3160c
+size 5129

runs/Oct26_08-01-52_RLHF000/events.out.tfevents.1729929739.RLHF000.222017.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35834f9ed8e1e2f807d7382117ff2319a1f7373bf4a143e41f0b9085b39e509e
+size 40739

runs/Oct26_08-01-52_RLHF000/events.out.tfevents.1729964586.RLHF000.222017.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8004aa31687b57a78dabf8077b9977829e8ec42aaac1dd1acb1e47d9479ed9e5
+size 828

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.8416462149295507,
-    "train_runtime": 20662.0179,
+    "train_loss": 0.5401819272219315,
+    "train_runtime": 34352.758,
     "train_samples": 61135,
-    "train_samples_per_second": 2.959,
-    "train_steps_per_second": 0.046
+    "train_samples_per_second": 1.78,
+    "train_steps_per_second": 0.015
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9994767137624281,
+  "epoch": 0.9990186457311089,
   "eval_steps": 100,
-  "global_step": 955,
+  "global_step": 509,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0,
-      "learning_rate": 5.208333333333333e-09,
-      "logits/chosen": -2.919764995574951,
-      "logits/rejected": -2.686896800994873,
-      "logps/chosen": -229.94229125976562,
-      "logps/rejected": -214.70114135742188,
-      "loss": 1.1369,
+      "learning_rate": 9.803921568627451e-09,
+      "logits/chosen": -2.7483465671539307,
+      "logits/rejected": -2.739339828491211,
+      "logps/chosen": -287.5325927734375,
+      "logps/rejected": -235.635986328125,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -23,1496 +23,802 @@
       "step": 1
     },
     {
-      "epoch": 0.01,
-      "learning_rate": 5.208333333333333e-08,
-      "logits/chosen": -2.680727005004883,
-      "logits/rejected": -2.7090559005737305,
-      "logps/chosen": -295.7759094238281,
-      "logps/rejected": -250.66514587402344,
-      "loss": 1.1367,
-      "rewards/accuracies": 0.5208333134651184,
-      "rewards/chosen": 0.0007360066520050168,
-      "rewards/margins": 0.0012527304934337735,
-      "rewards/rejected": -0.0005167239578440785,
+      "epoch": 0.02,
+      "learning_rate": 9.80392156862745e-08,
+      "logits/chosen": -2.709578037261963,
+      "logits/rejected": -2.7113540172576904,
+      "logps/chosen": -260.56292724609375,
+      "logps/rejected": -256.438232421875,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.4194444417953491,
+      "rewards/chosen": 0.00014394157915376127,
+      "rewards/margins": 1.0432106591906631e-06,
+      "rewards/rejected": 0.00014289839600678533,
       "step": 10
     },
     {
-      "epoch": 0.02,
-      "learning_rate": 1.0416666666666667e-07,
-      "logits/chosen": -2.6195316314697266,
-      "logits/rejected": -2.625615358352661,
-      "logps/chosen": -271.3199462890625,
-      "logps/rejected": -246.9070587158203,
-      "loss": 1.1366,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": 0.00032243202440440655,
-      "rewards/margins": 0.00047141723916865885,
-      "rewards/rejected": -0.0001489851565565914,
+      "epoch": 0.04,
+      "learning_rate": 1.96078431372549e-07,
+      "logits/chosen": -2.728665828704834,
+      "logits/rejected": -2.7061820030212402,
+      "logps/chosen": -280.0662536621094,
+      "logps/rejected": -254.76626586914062,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5724999904632568,
+      "rewards/chosen": -4.974007424607407e-06,
+      "rewards/margins": 0.0005589541979134083,
+      "rewards/rejected": -0.0005639282753691077,
       "step": 20
     },
     {
-      "epoch": 0.03,
-      "learning_rate": 1.5624999999999999e-07,
-      "logits/chosen": -2.7032415866851807,
-      "logits/rejected": -2.6663870811462402,
-      "logps/chosen": -278.2927551269531,
-      "logps/rejected": -254.49044799804688,
-      "loss": 1.1358,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0014447562862187624,
-      "rewards/margins": 0.001622636104002595,
-      "rewards/rejected": -0.0001778797450242564,
+      "epoch": 0.06,
+      "learning_rate": 2.941176470588235e-07,
+      "logits/chosen": -2.7290821075439453,
+      "logits/rejected": -2.742999315261841,
+      "logps/chosen": -279.2391357421875,
+      "logps/rejected": -253.37265014648438,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.6349999904632568,
+      "rewards/chosen": 0.0049138437025249004,
+      "rewards/margins": 0.007674422115087509,
+      "rewards/rejected": -0.002760578179731965,
       "step": 30
     },
     {
-      "epoch": 0.04,
-      "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.65032696723938,
-      "logits/rejected": -2.6380372047424316,
-      "logps/chosen": -273.85882568359375,
-      "logps/rejected": -237.75418090820312,
-      "loss": 1.1334,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.0036526708863675594,
-      "rewards/margins": 0.006543067749589682,
-      "rewards/rejected": -0.002890397561714053,
+      "epoch": 0.08,
+      "learning_rate": 3.92156862745098e-07,
+      "logits/chosen": -2.7134017944335938,
+      "logits/rejected": -2.698641777038574,
+      "logps/chosen": -274.20147705078125,
+      "logps/rejected": -255.8253936767578,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.6924999952316284,
+      "rewards/chosen": 0.0260241087526083,
+      "rewards/margins": 0.026919733732938766,
+      "rewards/rejected": -0.0008956241654232144,
       "step": 40
     },
     {
-      "epoch": 0.05,
-      "learning_rate": 2.604166666666667e-07,
-      "logits/chosen": -2.6743686199188232,
-      "logits/rejected": -2.638240337371826,
-      "logps/chosen": -296.05084228515625,
-      "logps/rejected": -274.6942138671875,
-      "loss": 1.1279,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.00925111211836338,
-      "rewards/margins": 0.01603672280907631,
-      "rewards/rejected": -0.0067856102250516415,
+      "epoch": 0.1,
+      "learning_rate": 4.901960784313725e-07,
+      "logits/chosen": -2.6435346603393555,
+      "logits/rejected": -2.6110424995422363,
+      "logps/chosen": -302.06768798828125,
+      "logps/rejected": -261.10919189453125,
+      "loss": 0.6612,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.023571131750941277,
+      "rewards/margins": 0.07649616152048111,
+      "rewards/rejected": -0.05292503535747528,
       "step": 50
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 3.1249999999999997e-07,
-      "logits/chosen": -2.6307342052459717,
-      "logits/rejected": -2.633100986480713,
-      "logps/chosen": -285.3183288574219,
-      "logps/rejected": -274.36505126953125,
-      "loss": 1.1188,
+      "epoch": 0.12,
+      "learning_rate": 4.995237599803335e-07,
+      "logits/chosen": -2.6205055713653564,
+      "logits/rejected": -2.5843255519866943,
+      "logps/chosen": -300.914306640625,
+      "logps/rejected": -286.0216064453125,
+      "loss": 0.6451,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.024691741913557053,
-      "rewards/margins": 0.02844650112092495,
-      "rewards/rejected": -0.0037547596730291843,
+      "rewards/chosen": -0.05583832785487175,
+      "rewards/margins": 0.11994686722755432,
+      "rewards/rejected": -0.17578519880771637,
       "step": 60
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 3.645833333333333e-07,
-      "logits/chosen": -2.649703025817871,
-      "logits/rejected": -2.6850831508636475,
-      "logps/chosen": -311.02667236328125,
-      "logps/rejected": -290.6634521484375,
-      "loss": 1.0997,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.03477492183446884,
-      "rewards/margins": 0.05690314620733261,
-      "rewards/rejected": -0.02212822437286377,
+      "epoch": 0.14,
+      "learning_rate": 4.978798275112142e-07,
+      "logits/chosen": -2.607668161392212,
+      "logits/rejected": -2.568187952041626,
+      "logps/chosen": -308.4685974121094,
+      "logps/rejected": -305.6259460449219,
+      "loss": 0.6212,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1777888685464859,
+      "rewards/margins": 0.19118839502334595,
+      "rewards/rejected": -0.3689771890640259,
       "step": 70
     },
     {
-      "epoch": 0.08,
-      "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.523608446121216,
-      "logits/rejected": -2.4657652378082275,
-      "logps/chosen": -304.8270568847656,
-      "logps/rejected": -282.1708068847656,
-      "loss": 1.0646,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.04266184940934181,
-      "rewards/margins": 0.1203114241361618,
-      "rewards/rejected": -0.1629732847213745,
+      "epoch": 0.16,
+      "learning_rate": 4.950700530747689e-07,
+      "logits/chosen": -2.6067116260528564,
+      "logits/rejected": -2.5767879486083984,
+      "logps/chosen": -300.19488525390625,
+      "logps/rejected": -295.8065185546875,
+      "loss": 0.6196,
+      "rewards/accuracies": 0.6850000023841858,
+      "rewards/chosen": -0.13195012509822845,
+      "rewards/margins": 0.25833892822265625,
+      "rewards/rejected": -0.3902890384197235,
       "step": 80
     },
     {
-      "epoch": 0.09,
-      "learning_rate": 4.6874999999999996e-07,
-      "logits/chosen": -2.5294718742370605,
-      "logits/rejected": -2.5001423358917236,
-      "logps/chosen": -293.15252685546875,
-      "logps/rejected": -285.61920166015625,
-      "loss": 1.0475,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.012959107756614685,
-      "rewards/margins": 0.17945662140846252,
-      "rewards/rejected": -0.1924157440662384,
+      "epoch": 0.18,
+      "learning_rate": 4.911076517558622e-07,
+      "logits/chosen": -2.5809831619262695,
+      "logits/rejected": -2.555103302001953,
+      "logps/chosen": -325.28692626953125,
+      "logps/rejected": -330.8323974609375,
+      "loss": 0.5844,
+      "rewards/accuracies": 0.7300000190734863,
+      "rewards/chosen": -0.21861158311367035,
+      "rewards/margins": 0.3220059275627136,
+      "rewards/rejected": -0.5406175851821899,
       "step": 90
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 4.999732492681437e-07,
-      "logits/chosen": -2.480851650238037,
-      "logits/rejected": -2.468801259994507,
-      "logps/chosen": -335.23480224609375,
-      "logps/rejected": -339.78021240234375,
-      "loss": 1.0407,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.2980949878692627,
-      "rewards/margins": 0.177928164601326,
-      "rewards/rejected": -0.4760231375694275,
+      "epoch": 0.2,
+      "learning_rate": 4.860112597371772e-07,
+      "logits/chosen": -2.5413742065429688,
+      "logits/rejected": -2.5363407135009766,
+      "logps/chosen": -295.8542175292969,
+      "logps/rejected": -310.6338195800781,
+      "loss": 0.5764,
+      "rewards/accuracies": 0.6675000190734863,
+      "rewards/chosen": -0.26630619168281555,
+      "rewards/margins": 0.3358945846557617,
+      "rewards/rejected": -0.6022006869316101,
       "step": 100
     },
     {
-      "epoch": 0.1,
-      "eval_logits/chosen": -2.5232999324798584,
-      "eval_logits/rejected": -2.486931324005127,
-      "eval_logps/chosen": -320.90283203125,
-      "eval_logps/rejected": -321.2205810546875,
-      "eval_loss": 1.0318840742111206,
-      "eval_rewards/accuracies": 0.6845238208770752,
-      "eval_rewards/chosen": -0.36793097853660583,
-      "eval_rewards/margins": 0.2264895737171173,
-      "eval_rewards/rejected": -0.5944206118583679,
-      "eval_runtime": 243.2829,
-      "eval_samples_per_second": 8.221,
-      "eval_steps_per_second": 0.259,
+      "epoch": 0.2,
+      "eval_logits/chosen": -2.4791219234466553,
+      "eval_logits/rejected": -2.4360005855560303,
+      "eval_logps/chosen": -313.6502990722656,
+      "eval_logps/rejected": -340.86053466796875,
+      "eval_loss": 0.5828901529312134,
+      "eval_rewards/accuracies": 0.6931137442588806,
+      "eval_rewards/chosen": -0.3592246174812317,
+      "eval_rewards/margins": 0.40203189849853516,
+      "eval_rewards/rejected": -0.7612565159797668,
+      "eval_runtime": 494.2516,
+      "eval_samples_per_second": 4.047,
+      "eval_steps_per_second": 0.338,
       "step": 100
     },
     {
-      "epoch": 0.12,
-      "learning_rate": 4.996723692767926e-07,
-      "logits/chosen": -2.2419214248657227,
-      "logits/rejected": -2.1671438217163086,
-      "logps/chosen": -305.33355712890625,
-      "logps/rejected": -294.58404541015625,
-      "loss": 1.0052,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.4501020312309265,
-      "rewards/margins": 0.32556745409965515,
-      "rewards/rejected": -0.7756695747375488,
+      "epoch": 0.22,
+      "learning_rate": 4.798048466485017e-07,
+      "logits/chosen": -2.0916123390197754,
+      "logits/rejected": -2.1291110515594482,
+      "logps/chosen": -337.0193786621094,
+      "logps/rejected": -372.4815368652344,
+      "loss": 0.5665,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.6119796633720398,
+      "rewards/margins": 0.5584384799003601,
+      "rewards/rejected": -1.1704181432724,
       "step": 110
     },
     {
-      "epoch": 0.13,
-      "learning_rate": 4.990375746213598e-07,
-      "logits/chosen": -1.2528715133666992,
-      "logits/rejected": -1.0699832439422607,
-      "logps/chosen": -357.62774658203125,
-      "logps/rejected": -348.74688720703125,
-      "loss": 0.9576,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.5205128788948059,
-      "rewards/margins": 0.43326228857040405,
-      "rewards/rejected": -0.9537751078605652,
+      "epoch": 0.24,
+      "learning_rate": 4.725176028314541e-07,
+      "logits/chosen": -1.8370585441589355,
+      "logits/rejected": -1.7712280750274658,
+      "logps/chosen": -370.1864318847656,
+      "logps/rejected": -398.8289794921875,
+      "loss": 0.56,
+      "rewards/accuracies": 0.7350000143051147,
+      "rewards/chosen": -0.8116917610168457,
+      "rewards/margins": 0.6380540728569031,
+      "rewards/rejected": -1.449745774269104,
       "step": 120
     },
     {
-      "epoch": 0.14,
-      "learning_rate": 4.980697142834314e-07,
-      "logits/chosen": -0.781643271446228,
-      "logits/rejected": -0.5821924209594727,
-      "logps/chosen": -406.4781799316406,
-      "logps/rejected": -411.183837890625,
-      "loss": 0.9475,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.8618149757385254,
-      "rewards/margins": 0.5553635954856873,
-      "rewards/rejected": -1.4171785116195679,
+      "epoch": 0.26,
+      "learning_rate": 4.641838020498713e-07,
+      "logits/chosen": -1.7485500574111938,
+      "logits/rejected": -1.5671393871307373,
+      "logps/chosen": -380.29913330078125,
+      "logps/rejected": -424.1035461425781,
+      "loss": 0.5461,
+      "rewards/accuracies": 0.7200000286102295,
+      "rewards/chosen": -0.8717474937438965,
+      "rewards/margins": 0.6444628834724426,
+      "rewards/rejected": -1.5162103176116943,
       "step": 130
     },
     {
-      "epoch": 0.15,
-      "learning_rate": 4.967700826904229e-07,
-      "logits/chosen": -0.470319926738739,
-      "logits/rejected": -0.3660030961036682,
-      "logps/chosen": -321.44873046875,
-      "logps/rejected": -374.0142517089844,
-      "loss": 0.8765,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8166507482528687,
-      "rewards/margins": 0.6684588193893433,
-      "rewards/rejected": -1.485109567642212,
+      "epoch": 0.27,
+      "learning_rate": 4.5484264029156733e-07,
+      "logits/chosen": -1.9667887687683105,
+      "logits/rejected": -1.6983026266098022,
+      "logps/chosen": -322.9972839355469,
+      "logps/rejected": -379.5963134765625,
+      "loss": 0.5416,
+      "rewards/accuracies": 0.7149999737739563,
+      "rewards/chosen": -0.6348860263824463,
+      "rewards/margins": 0.6040786504745483,
+      "rewards/rejected": -1.2389646768569946,
       "step": 140
     },
     {
-      "epoch": 0.16,
-      "learning_rate": 4.951404179843962e-07,
-      "logits/chosen": -0.061715979129076004,
-      "logits/rejected": -0.19356076419353485,
-      "logps/chosen": -405.5045471191406,
-      "logps/rejected": -456.53271484375,
-      "loss": 0.9764,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.2846033573150635,
-      "rewards/margins": 0.43445801734924316,
-      "rewards/rejected": -1.7190614938735962,
+      "epoch": 0.29,
+      "learning_rate": 4.445380514196192e-07,
+      "logits/chosen": -1.2058897018432617,
+      "logits/rejected": -0.9969528317451477,
+      "logps/chosen": -379.3441467285156,
+      "logps/rejected": -449.9009704589844,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.9173200726509094,
+      "rewards/margins": 0.7758927941322327,
+      "rewards/rejected": -1.6932127475738525,
       "step": 150
     },
     {
-      "epoch": 0.17,
-      "learning_rate": 4.931828996974498e-07,
-      "logits/chosen": -1.1201887130737305,
-      "logits/rejected": -0.6151416897773743,
-      "logps/chosen": -380.142822265625,
-      "logps/rejected": -427.4412536621094,
-      "loss": 0.8888,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -0.8433526754379272,
-      "rewards/margins": 0.5253406763076782,
-      "rewards/rejected": -1.368693232536316,
+      "epoch": 0.31,
+      "learning_rate": 4.33318500540218e-07,
+      "logits/chosen": -1.7521625757217407,
+      "logits/rejected": -1.4877443313598633,
+      "logps/chosen": -356.1580810546875,
+      "logps/rejected": -389.0058288574219,
+      "loss": 0.5183,
+      "rewards/accuracies": 0.7850000262260437,
+      "rewards/chosen": -0.6841800212860107,
+      "rewards/margins": 0.7851129174232483,
+      "rewards/rejected": -1.4692928791046143,
       "step": 160
     },
     {
-      "epoch": 0.18,
-      "learning_rate": 4.909001458367866e-07,
-      "logits/chosen": -0.5463464260101318,
-      "logits/rejected": -0.36673182249069214,
-      "logps/chosen": -386.74749755859375,
-      "logps/rejected": -429.04156494140625,
-      "loss": 0.8703,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.1368268728256226,
-      "rewards/margins": 0.681709885597229,
-      "rewards/rejected": -1.8185367584228516,
+      "epoch": 0.33,
+      "learning_rate": 4.2123675605892985e-07,
+      "logits/chosen": -1.6861900091171265,
+      "logits/rejected": -1.4684306383132935,
+      "logps/chosen": -379.7774658203125,
+      "logps/rejected": -437.3900451660156,
+      "loss": 0.5146,
+      "rewards/accuracies": 0.7300000190734863,
+      "rewards/chosen": -0.8159699440002441,
+      "rewards/margins": 0.7220683097839355,
+      "rewards/rejected": -1.5380383729934692,
       "step": 170
     },
     {
-      "epoch": 0.19,
-      "learning_rate": 4.882952093833627e-07,
-      "logits/chosen": 0.5949804186820984,
-      "logits/rejected": 0.9649137258529663,
-      "logps/chosen": -385.0345764160156,
-      "logps/rejected": -451.2206115722656,
-      "loss": 0.8344,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.35834801197052,
-      "rewards/margins": 0.8335322141647339,
-      "rewards/rejected": -2.191880464553833,
+      "epoch": 0.35,
+      "learning_rate": 4.0834964149744333e-07,
+      "logits/chosen": -1.3343206644058228,
+      "logits/rejected": -1.0179518461227417,
+      "logps/chosen": -358.3331298828125,
+      "logps/rejected": -399.9204406738281,
+      "loss": 0.5536,
+      "rewards/accuracies": 0.7074999809265137,
+      "rewards/chosen": -0.8257815837860107,
+      "rewards/margins": 0.7000215649604797,
+      "rewards/rejected": -1.5258032083511353,
       "step": 180
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 4.853715742087946e-07,
-      "logits/chosen": 0.7401331663131714,
-      "logits/rejected": 1.3593542575836182,
-      "logps/chosen": -480.5682067871094,
-      "logps/rejected": -533.142333984375,
-      "loss": 0.8896,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -2.017334461212158,
-      "rewards/margins": 0.7530598044395447,
-      "rewards/rejected": -2.7703945636749268,
+      "epoch": 0.37,
+      "learning_rate": 3.947177682380738e-07,
+      "logits/chosen": -1.2010215520858765,
+      "logits/rejected": -0.8926857709884644,
+      "logps/chosen": -375.1010437011719,
+      "logps/rejected": -433.2417297363281,
+      "loss": 0.5309,
+      "rewards/accuracies": 0.7425000071525574,
+      "rewards/chosen": -0.7876387238502502,
+      "rewards/margins": 0.7681831121444702,
+      "rewards/rejected": -1.5558221340179443,
       "step": 190
     },
     {
-      "epoch": 0.21,
-      "learning_rate": 4.821331504159906e-07,
-      "logits/chosen": 0.07484010607004166,
-      "logits/rejected": 0.5299566388130188,
-      "logps/chosen": -432.9810485839844,
-      "logps/rejected": -490.89599609375,
-      "loss": 0.8874,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.4631267786026,
-      "rewards/margins": 0.6909047365188599,
-      "rewards/rejected": -2.154031276702881,
+      "epoch": 0.39,
+      "learning_rate": 3.804052504529933e-07,
+      "logits/chosen": -1.1186742782592773,
+      "logits/rejected": -0.7032889723777771,
+      "logps/chosen": -351.2778625488281,
+      "logps/rejected": -416.71820068359375,
+      "loss": 0.5169,
+      "rewards/accuracies": 0.7475000023841858,
+      "rewards/chosen": -0.7259469032287598,
+      "rewards/margins": 0.874809741973877,
+      "rewards/rejected": -1.6007568836212158,
       "step": 200
     },
     {
-      "epoch": 0.21,
-      "eval_logits/chosen": -0.9007886648178101,
-      "eval_logits/rejected": -0.44379544258117676,
-      "eval_logps/chosen": -366.1669616699219,
-      "eval_logps/rejected": -415.2938537597656,
-      "eval_loss": 0.8947122097015381,
-      "eval_rewards/accuracies": 0.7480158805847168,
-      "eval_rewards/chosen": -0.8205717206001282,
-      "eval_rewards/margins": 0.7145815491676331,
-      "eval_rewards/rejected": -1.5351535081863403,
-      "eval_runtime": 244.6801,
-      "eval_samples_per_second": 8.174,
-      "eval_steps_per_second": 0.257,
+      "epoch": 0.39,
+      "eval_logits/chosen": -1.201006293296814,
+      "eval_logits/rejected": -0.8443379402160645,
+      "eval_logps/chosen": -366.2012023925781,
+      "eval_logps/rejected": -426.77203369140625,
+      "eval_loss": 0.531209409236908,
+      "eval_rewards/accuracies": 0.7065868377685547,
+      "eval_rewards/chosen": -0.8847335577011108,
+      "eval_rewards/margins": 0.7356376647949219,
+      "eval_rewards/rejected": -1.6203712224960327,
+      "eval_runtime": 494.1792,
+      "eval_samples_per_second": 4.047,
+      "eval_steps_per_second": 0.338,
       "step": 200
     },
     {
-      "epoch": 0.22,
-      "learning_rate": 4.785842691097342e-07,
-      "logits/chosen": -0.7449830770492554,
-      "logits/rejected": -0.08534111082553864,
-      "logps/chosen": -388.907470703125,
-      "logps/rejected": -403.8260498046875,
-      "loss": 0.8806,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -0.8363178968429565,
-      "rewards/margins": 0.6629363298416138,
-      "rewards/rejected": -1.4992539882659912,
+      "epoch": 0.41,
+      "learning_rate": 3.654794035589483e-07,
+      "logits/chosen": -0.9955520629882812,
+      "logits/rejected": -0.5436328649520874,
+      "logps/chosen": -402.7477722167969,
+      "logps/rejected": -444.9473876953125,
+      "loss": 0.5126,
+      "rewards/accuracies": 0.7225000262260437,
+      "rewards/chosen": -1.0243951082229614,
+      "rewards/margins": 0.7689486742019653,
+      "rewards/rejected": -1.7933436632156372,
       "step": 210
     },
     {
-      "epoch": 0.23,
-      "learning_rate": 4.7472967660421603e-07,
-      "logits/chosen": 0.45267248153686523,
-      "logits/rejected": 0.8757249116897583,
-      "logps/chosen": -412.01007080078125,
-      "logps/rejected": -479.5518493652344,
-      "loss": 0.8932,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2247607707977295,
-      "rewards/margins": 0.8075494766235352,
-      "rewards/rejected": -2.0323100090026855,
+      "epoch": 0.43,
+      "learning_rate": 3.5001042761570826e-07,
+      "logits/chosen": -0.7878814935684204,
+      "logits/rejected": -0.33438754081726074,
+      "logps/chosen": -379.41448974609375,
+      "logps/rejected": -452.28009033203125,
+      "loss": 0.5159,
+      "rewards/accuracies": 0.7475000023841858,
+      "rewards/chosen": -1.0701900720596313,
+      "rewards/margins": 0.8491780161857605,
+      "rewards/rejected": -1.919368028640747,
       "step": 220
     },
     {
-      "epoch": 0.24,
-      "learning_rate": 4.705745280752585e-07,
-      "logits/chosen": 0.03533775731921196,
-      "logits/rejected": 0.6978858709335327,
-      "logps/chosen": -428.5148010253906,
-      "logps/rejected": -484.0467834472656,
-      "loss": 0.8581,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.3480390310287476,
-      "rewards/margins": 1.0059764385223389,
-      "rewards/rejected": -2.354015588760376,
+      "epoch": 0.45,
+      "learning_rate": 3.34071077157304e-07,
+      "logits/chosen": -0.6851831078529358,
+      "logits/rejected": -0.29147180914878845,
+      "logps/chosen": -360.47869873046875,
+      "logps/rejected": -406.3958740234375,
+      "loss": 0.5399,
+      "rewards/accuracies": 0.7149999737739563,
+      "rewards/chosen": -0.9100778698921204,
+      "rewards/margins": 0.7056692242622375,
+      "rewards/rejected": -1.6157469749450684,
       "step": 230
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 4.6612438066572555e-07,
-      "logits/chosen": -0.08093588799238205,
-      "logits/rejected": 0.7299788594245911,
-      "logps/chosen": -392.78790283203125,
-      "logps/rejected": -432.1263122558594,
-      "loss": 0.8099,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.1553189754486084,
-      "rewards/margins": 0.9055362939834595,
-      "rewards/rejected": -2.0608553886413574,
+      "epoch": 0.47,
+      "learning_rate": 3.1773631900892204e-07,
+      "logits/chosen": -0.6293848752975464,
+      "logits/rejected": -0.2972988784313202,
+      "logps/chosen": -364.2557067871094,
+      "logps/rejected": -426.8414306640625,
+      "loss": 0.5184,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.945137083530426,
+      "rewards/margins": 0.7834777235984802,
+      "rewards/rejected": -1.7286149263381958,
       "step": 240
     },
     {
-      "epoch": 0.26,
-      "learning_rate": 4.6138518605333664e-07,
-      "logits/chosen": 0.8494974970817566,
-      "logits/rejected": 1.0580047369003296,
-      "logps/chosen": -410.2496032714844,
-      "logps/rejected": -510.062744140625,
-      "loss": 0.857,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.4266130924224854,
-      "rewards/margins": 0.8598226308822632,
-      "rewards/rejected": -2.286435604095459,
+      "epoch": 0.49,
+      "learning_rate": 3.0108297969883103e-07,
+      "logits/chosen": -0.6830095052719116,
+      "logits/rejected": -0.20727473497390747,
+      "logps/chosen": -377.15960693359375,
+      "logps/rejected": -440.8514709472656,
+      "loss": 0.5199,
+      "rewards/accuracies": 0.7475000023841858,
+      "rewards/chosen": -0.9253360033035278,
+      "rewards/margins": 0.7137148380279541,
+      "rewards/rejected": -1.6390507221221924,
       "step": 250
     },
     {
-      "epoch": 0.27,
-      "learning_rate": 4.5636328249082514e-07,
-      "logits/chosen": 0.6665963530540466,
-      "logits/rejected": 1.3317415714263916,
-      "logps/chosen": -429.8251037597656,
-      "logps/rejected": -490.12237548828125,
-      "loss": 0.8674,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.4576373100280762,
-      "rewards/margins": 0.86492520570755,
-      "rewards/rejected": -2.3225626945495605,
+      "epoch": 0.51,
+      "learning_rate": 2.8418938412365013e-07,
+      "logits/chosen": -0.595008909702301,
+      "logits/rejected": -0.22117982804775238,
+      "logps/chosen": -378.3102722167969,
+      "logps/rejected": -421.2056884765625,
+      "loss": 0.5259,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.0280470848083496,
+      "rewards/margins": 0.6548060774803162,
+      "rewards/rejected": -1.682853102684021,
       "step": 260
     },
     {
-      "epoch": 0.28,
-      "learning_rate": 4.510653863290871e-07,
-      "logits/chosen": -0.21562974154949188,
-      "logits/rejected": 0.4239919185638428,
-      "logps/chosen": -415.57073974609375,
-      "logps/rejected": -461.1295471191406,
-      "loss": 0.8284,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.1371711492538452,
-      "rewards/margins": 0.9719129800796509,
-      "rewards/rejected": -2.109084367752075,
+      "epoch": 0.53,
+      "learning_rate": 2.671349871664101e-07,
+      "logits/chosen": -0.4738517105579376,
+      "logits/rejected": -0.06301561743021011,
+      "logps/chosen": -391.0889892578125,
+      "logps/rejected": -433.60174560546875,
+      "loss": 0.4996,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.925932765007019,
+      "rewards/margins": 0.8979344367980957,
+      "rewards/rejected": -1.8238672018051147,
       "step": 270
     },
     {
-      "epoch": 0.29,
-      "learning_rate": 4.4549858303465737e-07,
-      "logits/chosen": 0.07179277390241623,
-      "logits/rejected": 0.6087414026260376,
-      "logps/chosen": -405.73614501953125,
-      "logps/rejected": -490.55908203125,
-      "loss": 0.8244,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.145477056503296,
-      "rewards/margins": 0.9263471364974976,
-      "rewards/rejected": -2.071824312210083,
+      "epoch": 0.55,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": -0.29330724477767944,
+      "logits/rejected": 0.11182761192321777,
+      "logps/chosen": -400.1533203125,
+      "logps/rejected": -453.4571228027344,
+      "loss": 0.5108,
+      "rewards/accuracies": 0.7174999713897705,
+      "rewards/chosen": -1.1598564386367798,
+      "rewards/margins": 0.7635893821716309,
+      "rewards/rejected": -1.9234455823898315,
       "step": 280
     },
     {
-      "epoch": 0.3,
-      "learning_rate": 4.396703177135261e-07,
-      "logits/chosen": 0.4951688349246979,
-      "logits/rejected": 0.9030885696411133,
-      "logps/chosen": -423.0415954589844,
-      "logps/rejected": -468.3642578125,
-      "loss": 0.852,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.3801145553588867,
-      "rewards/margins": 0.780128002166748,
-      "rewards/rejected": -2.1602425575256348,
+      "epoch": 0.57,
+      "learning_rate": 2.3286501283358982e-07,
+      "logits/chosen": -0.049084682017564774,
+      "logits/rejected": 0.32071781158447266,
+      "logps/chosen": -421.474853515625,
+      "logps/rejected": -480.5507507324219,
+      "loss": 0.5107,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -1.2823936939239502,
+      "rewards/margins": 0.920534610748291,
+      "rewards/rejected": -2.202928304672241,
       "step": 290
     },
     {
-      "epoch": 0.31,
-      "learning_rate": 4.335883851539693e-07,
-      "logits/chosen": 0.6263070106506348,
-      "logits/rejected": 1.3198049068450928,
-      "logps/chosen": -407.26031494140625,
-      "logps/rejected": -485.51983642578125,
-      "loss": 0.8068,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.2382786273956299,
-      "rewards/margins": 1.1074926853179932,
-      "rewards/rejected": -2.345771074295044,
+      "epoch": 0.59,
+      "learning_rate": 2.1581061587634987e-07,
+      "logits/chosen": -0.3210409879684448,
+      "logits/rejected": 0.13426151871681213,
+      "logps/chosen": -392.66351318359375,
+      "logps/rejected": -457.4385681152344,
+      "loss": 0.5133,
+      "rewards/accuracies": 0.7825000286102295,
+      "rewards/chosen": -1.2225959300994873,
+      "rewards/margins": 0.9219253659248352,
+      "rewards/rejected": -2.1445212364196777,
       "step": 300
     },
     {
-      "epoch": 0.31,
-      "eval_logits/chosen": 0.8117178082466125,
-      "eval_logits/rejected": 1.4832301139831543,
-      "eval_logps/chosen": -430.772216796875,
-      "eval_logps/rejected": -509.59326171875,
-      "eval_loss": 0.8381595015525818,
-      "eval_rewards/accuracies": 0.7539682388305664,
-      "eval_rewards/chosen": -1.4666248559951782,
-      "eval_rewards/margins": 1.0115222930908203,
-      "eval_rewards/rejected": -2.478147029876709,
-      "eval_runtime": 243.7025,
-      "eval_samples_per_second": 8.207,
-      "eval_steps_per_second": 0.259,
+      "epoch": 0.59,
+      "eval_logits/chosen": -0.38526856899261475,
+      "eval_logits/rejected": 0.0459565594792366,
+      "eval_logps/chosen": -396.590576171875,
+      "eval_logps/rejected": -460.7764892578125,
+      "eval_loss": 0.5159304141998291,
+      "eval_rewards/accuracies": 0.7245509028434753,
+      "eval_rewards/chosen": -1.1886271238327026,
+      "eval_rewards/margins": 0.7717891931533813,
+      "eval_rewards/rejected": -1.9604166746139526,
+      "eval_runtime": 494.4328,
+      "eval_samples_per_second": 4.045,
+      "eval_steps_per_second": 0.338,
       "step": 300
     },
     {
-      "epoch": 0.32,
-      "learning_rate": 4.272609194017105e-07,
-      "logits/chosen": 0.9425480961799622,
-      "logits/rejected": 1.6004616022109985,
-      "logps/chosen": -424.9742126464844,
-      "logps/rejected": -530.65869140625,
-      "loss": 0.7827,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.427599549293518,
-      "rewards/margins": 1.1877686977386475,
-      "rewards/rejected": -2.615368127822876,
+      "epoch": 0.61,
+      "learning_rate": 1.9891702030116897e-07,
+      "logits/chosen": -0.6406633257865906,
+      "logits/rejected": 0.15507885813713074,
+      "logps/chosen": -384.56219482421875,
+      "logps/rejected": -443.3284912109375,
+      "loss": 0.5192,
+      "rewards/accuracies": 0.7599999904632568,
+      "rewards/chosen": -1.066334843635559,
+      "rewards/margins": 0.8297566175460815,
+      "rewards/rejected": -1.8960914611816406,
       "step": 310
     },
     {
-      "epoch": 0.33,
-      "learning_rate": 4.2069638288135547e-07,
-      "logits/chosen": 0.7390011548995972,
-      "logits/rejected": 1.4064347743988037,
-      "logps/chosen": -448.7197265625,
-      "logps/rejected": -525.7418823242188,
-      "loss": 0.827,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.670648217201233,
-      "rewards/margins": 0.8930233120918274,
-      "rewards/rejected": -2.563671112060547,
+      "epoch": 0.63,
+      "learning_rate": 1.8226368099107792e-07,
+      "logits/chosen": -0.6926136016845703,
+      "logits/rejected": -0.09604160487651825,
+      "logps/chosen": -414.7826232910156,
+      "logps/rejected": -454.5480041503906,
+      "loss": 0.5065,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -1.0457278490066528,
+      "rewards/margins": 0.7350744605064392,
+      "rewards/rejected": -1.7808022499084473,
       "step": 320
     },
     {
-      "epoch": 0.35,
-      "learning_rate": 4.139035550786494e-07,
-      "logits/chosen": 1.1555938720703125,
-      "logits/rejected": 1.553986668586731,
-      "logps/chosen": -460.4817810058594,
-      "logps/rejected": -495.1622009277344,
-      "loss": 0.8771,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.8090622425079346,
-      "rewards/margins": 0.8104559779167175,
-      "rewards/rejected": -2.619518280029297,
+      "epoch": 0.65,
+      "learning_rate": 1.6592892284269594e-07,
+      "logits/chosen": -0.5141594409942627,
+      "logits/rejected": 0.11050853878259659,
+      "logps/chosen": -402.63348388671875,
+      "logps/rejected": -431.8319091796875,
+      "loss": 0.5093,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.0640606880187988,
+      "rewards/margins": 0.7925867438316345,
+      "rewards/rejected": -1.8566473722457886,
       "step": 330
     },
     {
-      "epoch": 0.36,
-      "learning_rate": 4.0689152079869306e-07,
-      "logits/chosen": 0.858076274394989,
-      "logits/rejected": 1.5555229187011719,
-      "logps/chosen": -441.11328125,
-      "logps/rejected": -489.79693603515625,
-      "loss": 0.8968,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.953595757484436,
-      "rewards/margins": 0.7021313905715942,
-      "rewards/rejected": -2.655726909637451,
+      "epoch": 0.67,
+      "learning_rate": 1.4998957238429172e-07,
+      "logits/chosen": -0.08297364413738251,
+      "logits/rejected": 0.21859808266162872,
+      "logps/chosen": -390.8412170410156,
+      "logps/rejected": -461.3310546875,
+      "loss": 0.505,
+      "rewards/accuracies": 0.7275000214576721,
+      "rewards/chosen": -1.190333604812622,
+      "rewards/margins": 0.8922053575515747,
+      "rewards/rejected": -2.0825393199920654,
       "step": 340
     },
     {
-      "epoch": 0.37,
-      "learning_rate": 3.99669658015821e-07,
-      "logits/chosen": 0.8753985166549683,
-      "logits/rejected": 1.0402195453643799,
-      "logps/chosen": -438.79534912109375,
-      "logps/rejected": -550.7193603515625,
-      "loss": 0.8261,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.6666584014892578,
-      "rewards/margins": 1.0503871440887451,
-      "rewards/rejected": -2.717045307159424,
+      "epoch": 0.69,
+      "learning_rate": 1.345205964410517e-07,
+      "logits/chosen": -0.539190948009491,
+      "logits/rejected": -0.053236301988363266,
+      "logps/chosen": -392.14385986328125,
+      "logps/rejected": -447.09844970703125,
+      "loss": 0.5125,
+      "rewards/accuracies": 0.7774999737739563,
+      "rewards/chosen": -0.9940242767333984,
+      "rewards/margins": 0.9291434288024902,
+      "rewards/rejected": -1.9231675863265991,
       "step": 350
     },
     {
-      "epoch": 0.38,
-      "learning_rate": 3.92247625331392e-07,
-      "logits/chosen": 0.5132797360420227,
-      "logits/rejected": 1.0654723644256592,
-      "logps/chosen": -414.5316467285156,
-      "logps/rejected": -473.7266540527344,
-      "loss": 0.8232,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.317946195602417,
-      "rewards/margins": 0.9921468496322632,
-      "rewards/rejected": -2.3100931644439697,
+      "epoch": 0.71,
+      "learning_rate": 1.1959474954700665e-07,
+      "logits/chosen": -0.6150873303413391,
+      "logits/rejected": -0.08470536023378372,
+      "logps/chosen": -377.5425109863281,
+      "logps/rejected": -434.1069030761719,
+      "loss": 0.5266,
+      "rewards/accuracies": 0.7599999904632568,
+      "rewards/chosen": -1.0171641111373901,
+      "rewards/margins": 0.7864332795143127,
+      "rewards/rejected": -1.803597092628479,
       "step": 360
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 3.846353490562664e-07,
-      "logits/chosen": 0.6082018613815308,
-      "logits/rejected": 1.0670816898345947,
-      "logps/chosen": -366.9786071777344,
-      "logps/rejected": -486.1373596191406,
-      "loss": 0.7788,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.2683700323104858,
-      "rewards/margins": 1.0532209873199463,
-      "rewards/rejected": -2.3215911388397217,
+      "epoch": 0.73,
+      "learning_rate": 1.0528223176192615e-07,
+      "logits/chosen": -0.464309424161911,
+      "logits/rejected": 0.11655576527118683,
+      "logps/chosen": -397.9951477050781,
+      "logps/rejected": -446.141845703125,
+      "loss": 0.4885,
+      "rewards/accuracies": 0.7350000143051147,
+      "rewards/chosen": -1.1220192909240723,
+      "rewards/margins": 0.7690063714981079,
+      "rewards/rejected": -1.8910256624221802,
       "step": 370
     },
     {
-      "epoch": 0.4,
-      "learning_rate": 3.768430099352445e-07,
-      "logits/chosen": 0.43745535612106323,
-      "logits/rejected": 1.5503931045532227,
-      "logps/chosen": -478.939208984375,
-      "logps/rejected": -556.5857543945312,
-      "loss": 0.7987,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.8233133554458618,
-      "rewards/margins": 1.1179938316345215,
-      "rewards/rejected": -2.9413070678710938,
+      "epoch": 0.75,
+      "learning_rate": 9.16503585025567e-08,
+      "logits/chosen": -0.3131292462348938,
+      "logits/rejected": 0.1059599220752716,
+      "logps/chosen": -398.6189880371094,
+      "logps/rejected": -455.5489807128906,
+      "loss": 0.4785,
+      "rewards/accuracies": 0.7774999737739563,
+      "rewards/chosen": -1.180424451828003,
+      "rewards/margins": 0.9602058529853821,
+      "rewards/rejected": -2.1406302452087402,
       "step": 380
     },
     {
-      "epoch": 0.41,
-      "learning_rate": 3.6888102953122304e-07,
-      "logits/chosen": 1.4241774082183838,
-      "logits/rejected": 2.120664596557617,
-      "logps/chosen": -464.5999450683594,
-      "logps/rejected": -538.8297729492188,
-      "loss": 0.7897,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.8883854150772095,
-      "rewards/margins": 1.1485675573349,
-      "rewards/rejected": -3.0369527339935303,
+      "epoch": 0.77,
+      "learning_rate": 7.876324394107017e-08,
+      "logits/chosen": -0.06371825933456421,
+      "logits/rejected": 0.4222162663936615,
+      "logps/chosen": -408.15203857421875,
+      "logps/rejected": -469.3525085449219,
+      "loss": 0.4945,
+      "rewards/accuracies": 0.7774999737739563,
+      "rewards/chosen": -1.2744272947311401,
+      "rewards/margins": 0.8693990111351013,
+      "rewards/rejected": -2.1438262462615967,
       "step": 390
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 3.607600562872785e-07,
-      "logits/chosen": 1.6316184997558594,
-      "logits/rejected": 2.457326650619507,
-      "logps/chosen": -501.25262451171875,
-      "logps/rejected": -549.1324462890625,
-      "loss": 0.7845,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.9686987400054932,
-      "rewards/margins": 0.8676248788833618,
-      "rewards/rejected": -2.8363232612609863,
+      "epoch": 0.79,
+      "learning_rate": 6.668149945978201e-08,
+      "logits/chosen": -0.4337286353111267,
+      "logits/rejected": 0.11450805515050888,
+      "logps/chosen": -406.1577453613281,
+      "logps/rejected": -468.1871337890625,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.7574999928474426,
+      "rewards/chosen": -1.204884648323059,
+      "rewards/margins": 0.9240193367004395,
+      "rewards/rejected": -2.128904104232788,
       "step": 400
     },
     {
-      "epoch": 0.42,
-      "eval_logits/chosen": 2.1264495849609375,
-      "eval_logits/rejected": 2.8754770755767822,
-      "eval_logps/chosen": -461.9886779785156,
-      "eval_logps/rejected": -553.6510009765625,
-      "eval_loss": 0.8208896517753601,
-      "eval_rewards/accuracies": 0.7519841194152832,
-      "eval_rewards/chosen": -1.7787890434265137,
-      "eval_rewards/margins": 1.1399353742599487,
-      "eval_rewards/rejected": -2.91872501373291,
-      "eval_runtime": 243.2973,
-      "eval_samples_per_second": 8.22,
-      "eval_steps_per_second": 0.259,
+      "epoch": 0.79,
+      "eval_logits/chosen": -0.2552393972873688,
+      "eval_logits/rejected": 0.20138485729694366,
+      "eval_logps/chosen": -402.1766357421875,
+      "eval_logps/rejected": -475.3639221191406,
+      "eval_loss": 0.5057728290557861,
+      "eval_rewards/accuracies": 0.7140718698501587,
+      "eval_rewards/chosen": -1.2444883584976196,
+      "eval_rewards/margins": 0.8618020415306091,
+      "eval_rewards/rejected": -2.106290578842163,
+      "eval_runtime": 493.9837,
+      "eval_samples_per_second": 4.049,
+      "eval_steps_per_second": 0.338,
       "step": 400
     },
     {
-      "epoch": 0.43,
-      "learning_rate": 3.5249095128531856e-07,
-      "logits/chosen": 1.401972770690918,
-      "logits/rejected": 2.366103410720825,
-      "logps/chosen": -486.5357971191406,
-      "logps/rejected": -573.39013671875,
-      "loss": 0.8379,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.7267777919769287,
-      "rewards/margins": 1.0969970226287842,
-      "rewards/rejected": -2.823775053024292,
+      "epoch": 0.8,
+      "learning_rate": 5.546194858038072e-08,
+      "logits/chosen": -0.3444100618362427,
+      "logits/rejected": 0.08428356051445007,
+      "logps/chosen": -419.0089111328125,
+      "logps/rejected": -482.5577392578125,
+      "loss": 0.488,
+      "rewards/accuracies": 0.7325000166893005,
+      "rewards/chosen": -1.1570134162902832,
+      "rewards/margins": 0.9088660478591919,
+      "rewards/rejected": -2.0658795833587646,
       "step": 410
     },
     {
-      "epoch": 0.44,
-      "learning_rate": 3.4408477372034736e-07,
-      "logits/chosen": 1.4769244194030762,
-      "logits/rejected": 2.5726444721221924,
-      "logps/chosen": -433.69610595703125,
-      "logps/rejected": -487.23077392578125,
-      "loss": 0.823,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.6958898305892944,
-      "rewards/margins": 0.8770249485969543,
-      "rewards/rejected": -2.5729150772094727,
+      "epoch": 0.82,
+      "learning_rate": 4.5157359708432626e-08,
+      "logits/chosen": -0.3363034129142761,
+      "logits/rejected": 0.1421819031238556,
+      "logps/chosen": -417.26116943359375,
+      "logps/rejected": -475.9188537597656,
+      "loss": 0.5012,
+      "rewards/accuracies": 0.7549999952316284,
+      "rewards/chosen": -1.1876376867294312,
+      "rewards/margins": 0.9119570255279541,
+      "rewards/rejected": -2.0995945930480957,
       "step": 420
     },
     {
-      "epoch": 0.45,
-      "learning_rate": 3.3555276610977276e-07,
-      "logits/chosen": 2.185957431793213,
-      "logits/rejected": 3.1616008281707764,
-      "logps/chosen": -451.7828063964844,
-      "logps/rejected": -523.4830932617188,
-      "loss": 0.8662,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.8958429098129272,
-      "rewards/margins": 0.9449018239974976,
-      "rewards/rejected": -2.840744733810425,
+      "epoch": 0.84,
+      "learning_rate": 3.581619795012874e-08,
+      "logits/chosen": -0.4450594186782837,
+      "logits/rejected": 0.03785795345902443,
+      "logps/chosen": -404.95281982421875,
+      "logps/rejected": -467.25531005859375,
+      "loss": 0.4861,
+      "rewards/accuracies": 0.7724999785423279,
+      "rewards/chosen": -1.1584584712982178,
+      "rewards/margins": 0.9622448086738586,
+      "rewards/rejected": -2.1207032203674316,
       "step": 430
     },
     {
-      "epoch": 0.46,
-      "learning_rate": 3.269063392575352e-07,
-      "logits/chosen": 2.3363919258117676,
-      "logits/rejected": 2.1066231727600098,
-      "logps/chosen": -447.093994140625,
-      "logps/rejected": -537.6717529296875,
-      "loss": 0.8362,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.8810195922851562,
-      "rewards/margins": 1.006998896598816,
-      "rewards/rejected": -2.8880181312561035,
+      "epoch": 0.86,
+      "learning_rate": 2.748239716854589e-08,
+      "logits/chosen": -0.31011733412742615,
+      "logits/rejected": 0.310569167137146,
+      "logps/chosen": -389.67132568359375,
+      "logps/rejected": -470.01104736328125,
+      "loss": 0.5105,
+      "rewards/accuracies": 0.7350000143051147,
+      "rewards/chosen": -1.1304560899734497,
+      "rewards/margins": 0.8861461877822876,
+      "rewards/rejected": -2.016602039337158,
       "step": 440
     },
     {
-      "epoch": 0.47,
-      "learning_rate": 3.1815705699316964e-07,
-      "logits/chosen": 1.551466464996338,
-      "logits/rejected": 1.8984363079071045,
-      "logps/chosen": -445.85894775390625,
-      "logps/rejected": -539.7463989257812,
-      "loss": 0.8323,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.817039132118225,
-      "rewards/margins": 1.14361572265625,
-      "rewards/rejected": -2.9606547355651855,
+      "epoch": 0.88,
+      "learning_rate": 2.0195153351498323e-08,
+      "logits/chosen": -0.3003827631473541,
+      "logits/rejected": 0.046957388520240784,
+      "logps/chosen": -412.5171203613281,
+      "logps/rejected": -481.26898193359375,
+      "loss": 0.5128,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.1812173128128052,
+      "rewards/margins": 0.8305546641349792,
+      "rewards/rejected": -2.0117719173431396,
       "step": 450
     },
     {
-      "epoch": 0.48,
-      "learning_rate": 3.0931662070620794e-07,
-      "logits/chosen": 0.859958827495575,
-      "logits/rejected": 1.7097549438476562,
-      "logps/chosen": -435.4864807128906,
-      "logps/rejected": -531.817138671875,
-      "loss": 0.8005,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.7045366764068604,
-      "rewards/margins": 1.0461620092391968,
-      "rewards/rejected": -2.7506985664367676,
+      "epoch": 0.9,
+      "learning_rate": 1.3988740262822846e-08,
+      "logits/chosen": -0.47582343220710754,
+      "logits/rejected": -0.11152289062738419,
+      "logps/chosen": -410.2917175292969,
+      "logps/rejected": -457.774658203125,
+      "loss": 0.5044,
+      "rewards/accuracies": 0.7649999856948853,
+      "rewards/chosen": -1.1460288763046265,
+      "rewards/margins": 0.8546761870384216,
+      "rewards/rejected": -2.0007050037384033,
       "step": 460
     },
     {
-      "epoch": 0.49,
-      "learning_rate": 3.003968536966078e-07,
-      "logits/chosen": 1.0430405139923096,
-      "logits/rejected": 1.8049592971801758,
-      "logps/chosen": -466.21612548828125,
-      "logps/rejected": -558.8511962890625,
-      "loss": 0.7768,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.6801469326019287,
-      "rewards/margins": 1.3035141229629517,
-      "rewards/rejected": -2.983660936355591,
+      "epoch": 0.92,
+      "learning_rate": 8.892348244137788e-09,
+      "logits/chosen": -0.5770422220230103,
+      "logits/rejected": -0.025662722066044807,
+      "logps/chosen": -372.98187255859375,
+      "logps/rejected": -467.86199951171875,
+      "loss": 0.4973,
+      "rewards/accuracies": 0.7200000286102295,
+      "rewards/chosen": -1.0886142253875732,
+      "rewards/margins": 0.8808639049530029,
+      "rewards/rejected": -1.9694780111312866,
       "step": 470
     },
     {
-      "epoch": 0.5,
-      "learning_rate": 2.9140968536213693e-07,
-      "logits/chosen": 1.5663492679595947,
-      "logits/rejected": 2.4222500324249268,
-      "logps/chosen": -411.6764221191406,
-      "logps/rejected": -506.650390625,
-      "loss": 0.8345,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.7893803119659424,
-      "rewards/margins": 0.9885958433151245,
-      "rewards/rejected": -2.7779765129089355,
+      "epoch": 0.94,
+      "learning_rate": 4.929946925231076e-09,
+      "logits/chosen": -0.5876446962356567,
+      "logits/rejected": -0.16365936398506165,
+      "logps/chosen": -400.3377685546875,
+      "logps/rejected": -455.9208068847656,
+      "loss": 0.5072,
+      "rewards/accuracies": 0.7024999856948853,
+      "rewards/chosen": -1.1451067924499512,
+      "rewards/margins": 0.7030719518661499,
+      "rewards/rejected": -1.848178744316101,
       "step": 480
     },
     {
-      "epoch": 0.51,
-      "learning_rate": 2.823671352438608e-07,
-      "logits/chosen": 1.7138950824737549,
-      "logits/rejected": 2.425443172454834,
-      "logps/chosen": -452.81439208984375,
-      "logps/rejected": -527.6519165039062,
-      "loss": 0.79,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.7711286544799805,
-      "rewards/margins": 1.0116374492645264,
-      "rewards/rejected": -2.782766103744507,
+      "epoch": 0.96,
+      "learning_rate": 2.1201724887858484e-09,
+      "logits/chosen": -0.4430970847606659,
+      "logits/rejected": 0.12594802677631378,
+      "logps/chosen": -409.6846008300781,
+      "logps/rejected": -458.5526428222656,
+      "loss": 0.4887,
+      "rewards/accuracies": 0.7574999928474426,
+      "rewards/chosen": -1.0775573253631592,
+      "rewards/margins": 0.9305427074432373,
+      "rewards/rejected": -2.0081000328063965,
       "step": 490
     },
     {
-      "epoch": 0.52,
-      "learning_rate": 2.73281296951072e-07,
-      "logits/chosen": 1.9489076137542725,
-      "logits/rejected": 2.5219855308532715,
-      "logps/chosen": -453.45367431640625,
-      "logps/rejected": -559.2427978515625,
-      "loss": 0.8323,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.8187179565429688,
-      "rewards/margins": 1.278545618057251,
-      "rewards/rejected": -3.0972630977630615,
-      "step": 500
-    },
-    {
-      "epoch": 0.52,
-      "eval_logits/chosen": 1.3306254148483276,
-      "eval_logits/rejected": 2.1307854652404785,
-      "eval_logps/chosen": -427.62841796875,
-      "eval_logps/rejected": -516.395263671875,
-      "eval_loss": 0.8331887722015381,
-      "eval_rewards/accuracies": 0.7440476417541504,
-      "eval_rewards/chosen": -1.4351868629455566,
-      "eval_rewards/margins": 1.1109802722930908,
-      "eval_rewards/rejected": -2.5461671352386475,
-      "eval_runtime": 243.742,
-      "eval_samples_per_second": 8.205,
-      "eval_steps_per_second": 0.258,
+      "epoch": 0.98,
+      "learning_rate": 4.762400196664518e-10,
+      "logits/chosen": -0.41937455534935,
+      "logits/rejected": -0.08660510927438736,
+      "logps/chosen": -385.8563232421875,
+      "logps/rejected": -454.9473571777344,
+      "loss": 0.4833,
+      "rewards/accuracies": 0.7699999809265137,
+      "rewards/chosen": -1.093034267425537,
+      "rewards/margins": 0.9196186661720276,
+      "rewards/rejected": -2.012652635574341,
       "step": 500
     },
-    {
-      "epoch": 0.53,
-      "learning_rate": 2.641643219871597e-07,
-      "logits/chosen": 1.4478816986083984,
-      "logits/rejected": 2.14026141166687,
-      "logps/chosen": -437.3523864746094,
-      "logps/rejected": -497.67791748046875,
-      "loss": 0.778,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.3602588176727295,
-      "rewards/margins": 1.100663423538208,
-      "rewards/rejected": -2.4609227180480957,
-      "step": 510
-    },
-    {
-      "epoch": 0.54,
-      "learning_rate": 2.550284034980507e-07,
-      "logits/chosen": 1.3803322315216064,
-      "logits/rejected": 2.3976683616638184,
-      "logps/chosen": -412.71685791015625,
-      "logps/rejected": -522.4058837890625,
-      "loss": 0.797,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.535652756690979,
-      "rewards/margins": 1.171526312828064,
-      "rewards/rejected": -2.707179069519043,
-      "step": 520
-    },
-    {
-      "epoch": 0.55,
-      "learning_rate": 2.4588575996495794e-07,
-      "logits/chosen": 2.3339693546295166,
-      "logits/rejected": 3.0342774391174316,
-      "logps/chosen": -474.455078125,
-      "logps/rejected": -575.391357421875,
-      "loss": 0.7824,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.9030723571777344,
-      "rewards/margins": 1.281085729598999,
-      "rewards/rejected": -3.1841578483581543,
-      "step": 530
-    },
-    {
-      "epoch": 0.57,
-      "learning_rate": 2.367486188632446e-07,
-      "logits/chosen": 2.5718750953674316,
-      "logits/rejected": 3.330522060394287,
-      "logps/chosen": -513.03955078125,
-      "logps/rejected": -665.7197875976562,
-      "loss": 0.8011,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.188791036605835,
-      "rewards/margins": 1.4658076763153076,
-      "rewards/rejected": -3.6545987129211426,
-      "step": 540
-    },
-    {
-      "epoch": 0.58,
-      "learning_rate": 2.276292003092593e-07,
-      "logits/chosen": 3.1903679370880127,
-      "logits/rejected": 4.050724983215332,
-      "logps/chosen": -495.56915283203125,
-      "logps/rejected": -606.86376953125,
-      "loss": 0.8026,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -2.2584142684936523,
-      "rewards/margins": 1.3304917812347412,
-      "rewards/rejected": -3.5889060497283936,
-      "step": 550
-    },
-    {
-      "epoch": 0.59,
-      "learning_rate": 2.185397007170141e-07,
-      "logits/chosen": 3.2466537952423096,
-      "logits/rejected": 3.8034489154815674,
-      "logps/chosen": -482.8548278808594,
-      "logps/rejected": -559.918701171875,
-      "loss": 0.8114,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.160890817642212,
-      "rewards/margins": 1.1038119792938232,
-      "rewards/rejected": -3.264702558517456,
-      "step": 560
-    },
-    {
-      "epoch": 0.6,
-      "learning_rate": 2.094922764865619e-07,
-      "logits/chosen": 2.1195971965789795,
-      "logits/rejected": 3.2914786338806152,
-      "logps/chosen": -474.29058837890625,
-      "logps/rejected": -559.2501831054688,
-      "loss": 0.8237,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.981508493423462,
-      "rewards/margins": 1.0219013690948486,
-      "rewards/rejected": -3.0034098625183105,
-      "step": 570
-    },
-    {
-      "epoch": 0.61,
-      "learning_rate": 2.0049902774588797e-07,
-      "logits/chosen": 2.163456916809082,
-      "logits/rejected": 3.159027576446533,
-      "logps/chosen": -483.96038818359375,
-      "logps/rejected": -568.642578125,
-      "loss": 0.817,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -2.100404977798462,
-      "rewards/margins": 1.2073109149932861,
-      "rewards/rejected": -3.307715654373169,
-      "step": 580
-    },
-    {
-      "epoch": 0.62,
-      "learning_rate": 1.9157198216806238e-07,
-      "logits/chosen": 1.664136528968811,
-      "logits/rejected": 2.9907596111297607,
-      "logps/chosen": -458.36456298828125,
-      "logps/rejected": -561.4249267578125,
-      "loss": 0.7903,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.8531945943832397,
-      "rewards/margins": 0.9679274559020996,
-      "rewards/rejected": -2.8211216926574707,
-      "step": 590
-    },
-    {
-      "epoch": 0.63,
-      "learning_rate": 1.8272307888529274e-07,
-      "logits/chosen": 1.441373348236084,
-      "logits/rejected": 3.0396907329559326,
-      "logps/chosen": -518.3441772460938,
-      "logps/rejected": -635.1202392578125,
-      "loss": 0.7677,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.9236825704574585,
-      "rewards/margins": 1.306647777557373,
-      "rewards/rejected": -3.230330228805542,
-      "step": 600
-    },
-    {
-      "epoch": 0.63,
-      "eval_logits/chosen": 1.8965519666671753,
-      "eval_logits/rejected": 3.3207030296325684,
-      "eval_logps/chosen": -503.2609558105469,
-      "eval_logps/rejected": -616.7921142578125,
-      "eval_loss": 0.7981351613998413,
-      "eval_rewards/accuracies": 0.7519841194152832,
-      "eval_rewards/chosen": -2.191512107849121,
-      "eval_rewards/margins": 1.358623743057251,
-      "eval_rewards/rejected": -3.550135612487793,
-      "eval_runtime": 243.1992,
-      "eval_samples_per_second": 8.224,
-      "eval_steps_per_second": 0.259,
-      "step": 600
-    },
-    {
-      "epoch": 0.64,
-      "learning_rate": 1.7396415252139288e-07,
-      "logits/chosen": 2.0666539669036865,
-      "logits/rejected": 3.6420180797576904,
-      "logps/chosen": -494.7054748535156,
-      "logps/rejected": -573.6849975585938,
-      "loss": 0.7714,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.1968560218811035,
-      "rewards/margins": 1.295644760131836,
-      "rewards/rejected": -3.4925007820129395,
-      "step": 610
-    },
-    {
-      "epoch": 0.65,
-      "learning_rate": 1.6530691736402316e-07,
-      "logits/chosen": 2.116316795349121,
-      "logits/rejected": 3.3095905780792236,
-      "logps/chosen": -510.40380859375,
-      "logps/rejected": -580.3692016601562,
-      "loss": 0.7844,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.3837578296661377,
-      "rewards/margins": 1.1186126470565796,
-      "rewards/rejected": -3.5023703575134277,
-      "step": 620
-    },
-    {
-      "epoch": 0.66,
-      "learning_rate": 1.5676295169786864e-07,
-      "logits/chosen": 2.523867130279541,
-      "logits/rejected": 3.438098192214966,
-      "logps/chosen": -483.1630859375,
-      "logps/rejected": -625.9393310546875,
-      "loss": 0.7686,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -2.2157483100891113,
-      "rewards/margins": 1.489645004272461,
-      "rewards/rejected": -3.7053933143615723,
-      "step": 630
-    },
-    {
-      "epoch": 0.67,
-      "learning_rate": 1.483436823197092e-07,
-      "logits/chosen": 1.7444801330566406,
-      "logits/rejected": 2.5841469764709473,
-      "logps/chosen": -479.9925842285156,
-      "logps/rejected": -581.1832885742188,
-      "loss": 0.7969,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.9535319805145264,
-      "rewards/margins": 1.2860063314437866,
-      "rewards/rejected": -3.2395381927490234,
-      "step": 640
-    },
-    {
-      "epoch": 0.68,
-      "learning_rate": 1.4006036925609243e-07,
-      "logits/chosen": 1.0596259832382202,
-      "logits/rejected": 2.4878382682800293,
-      "logps/chosen": -521.443115234375,
-      "logps/rejected": -584.9697875976562,
-      "loss": 0.7884,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.1512815952301025,
-      "rewards/margins": 1.0702219009399414,
-      "rewards/rejected": -3.221503734588623,
-      "step": 650
-    },
-    {
-      "epoch": 0.69,
-      "learning_rate": 1.319240907040458e-07,
-      "logits/chosen": 1.1169414520263672,
-      "logits/rejected": 1.6374781131744385,
-      "logps/chosen": -494.7315368652344,
-      "logps/rejected": -578.5076904296875,
-      "loss": 0.7997,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.9317266941070557,
-      "rewards/margins": 1.1828477382659912,
-      "rewards/rejected": -3.1145741939544678,
-      "step": 660
-    },
-    {
-      "epoch": 0.7,
-      "learning_rate": 1.239457282149695e-07,
-      "logits/chosen": 1.0661325454711914,
-      "logits/rejected": 2.232234477996826,
-      "logps/chosen": -471.725341796875,
-      "logps/rejected": -577.9931640625,
-      "loss": 0.791,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.8857122659683228,
-      "rewards/margins": 1.2069826126098633,
-      "rewards/rejected": -3.0926949977874756,
-      "step": 670
-    },
-    {
-      "epoch": 0.71,
-      "learning_rate": 1.1613595214152711e-07,
-      "logits/chosen": 1.3097771406173706,
-      "logits/rejected": 1.997799277305603,
-      "logps/chosen": -444.1643981933594,
-      "logps/rejected": -502.30548095703125,
-      "loss": 0.8287,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.826926827430725,
-      "rewards/margins": 0.8142975568771362,
-      "rewards/rejected": -2.6412243843078613,
-      "step": 680
-    },
-    {
-      "epoch": 0.72,
-      "learning_rate": 1.0850520736699362e-07,
-      "logits/chosen": 0.9592529535293579,
-      "logits/rejected": 2.325680732727051,
-      "logps/chosen": -430.89532470703125,
-      "logps/rejected": -537.284423828125,
-      "loss": 0.7625,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.6944091320037842,
-      "rewards/margins": 1.2595393657684326,
-      "rewards/rejected": -2.953948497772217,
-      "step": 690
-    },
-    {
-      "epoch": 0.73,
-      "learning_rate": 1.0106369933615042e-07,
-      "logits/chosen": 1.8058967590332031,
-      "logits/rejected": 3.336688280105591,
-      "logps/chosen": -486.33380126953125,
-      "logps/rejected": -616.1010131835938,
-      "loss": 0.7227,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.3213586807250977,
-      "rewards/margins": 1.3918098211288452,
-      "rewards/rejected": -3.7131690979003906,
-      "step": 700
-    },
-    {
-      "epoch": 0.73,
-      "eval_logits/chosen": 1.7450497150421143,
-      "eval_logits/rejected": 2.97623610496521,
-      "eval_logps/chosen": -507.26715087890625,
-      "eval_logps/rejected": -623.6929321289062,
-      "eval_loss": 0.7834469079971313,
-      "eval_rewards/accuracies": 0.7638888955116272,
-      "eval_rewards/chosen": -2.2315735816955566,
-      "eval_rewards/margins": 1.3875702619552612,
-      "eval_rewards/rejected": -3.6191442012786865,
-      "eval_runtime": 244.2063,
-      "eval_samples_per_second": 8.19,
-      "eval_steps_per_second": 0.258,
-      "step": 700
-    },
-    {
-      "epoch": 0.74,
-      "learning_rate": 9.382138040640714e-08,
-      "logits/chosen": 1.7643959522247314,
-      "logits/rejected": 3.0291295051574707,
-      "logps/chosen": -539.8101806640625,
-      "logps/rejected": -597.0071411132812,
-      "loss": 0.7477,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -2.3488943576812744,
-      "rewards/margins": 1.2158434391021729,
-      "rewards/rejected": -3.5647377967834473,
-      "step": 710
-    },
-    {
-      "epoch": 0.75,
-      "learning_rate": 8.678793653740632e-08,
-      "logits/chosen": 2.335305690765381,
-      "logits/rejected": 3.246683120727539,
-      "logps/chosen": -533.5028076171875,
-      "logps/rejected": -652.6378173828125,
-      "loss": 0.759,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -2.420844554901123,
-      "rewards/margins": 1.4402214288711548,
-      "rewards/rejected": -3.8610661029815674,
-      "step": 720
-    },
-    {
-      "epoch": 0.76,
-      "learning_rate": 7.997277433690983e-08,
-      "logits/chosen": 2.274118661880493,
-      "logits/rejected": 3.46764874458313,
-      "logps/chosen": -541.6676025390625,
-      "logps/rejected": -625.87744140625,
-      "loss": 0.7646,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.377727746963501,
-      "rewards/margins": 1.3614501953125,
-      "rewards/rejected": -3.73917818069458,
-      "step": 730
-    },
-    {
-      "epoch": 0.77,
-      "learning_rate": 7.338500848029602e-08,
-      "logits/chosen": 2.568730592727661,
-      "logits/rejected": 3.9025726318359375,
-      "logps/chosen": -562.5526123046875,
-      "logps/rejected": -680.5411376953125,
-      "loss": 0.7825,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.482536792755127,
-      "rewards/margins": 1.7319949865341187,
-      "rewards/rejected": -4.214531898498535,
-      "step": 740
-    },
-    {
-      "epoch": 0.78,
-      "learning_rate": 6.70334495204884e-08,
-      "logits/chosen": 2.1631178855895996,
-      "logits/rejected": 3.1983213424682617,
-      "logps/chosen": -532.3298950195312,
-      "logps/rejected": -670.1683349609375,
-      "loss": 0.7519,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.4897987842559814,
-      "rewards/margins": 1.4266437292099,
-      "rewards/rejected": -3.916442394256592,
-      "step": 750
-    },
-    {
-      "epoch": 0.8,
-      "learning_rate": 6.092659210462231e-08,
-      "logits/chosen": 2.325679302215576,
-      "logits/rejected": 2.9977734088897705,
-      "logps/chosen": -529.2430419921875,
-      "logps/rejected": -633.878173828125,
-      "loss": 0.7115,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -2.5113611221313477,
-      "rewards/margins": 1.3206489086151123,
-      "rewards/rejected": -3.832010269165039,
-      "step": 760
-    },
-    {
-      "epoch": 0.81,
-      "learning_rate": 5.507260361320737e-08,
-      "logits/chosen": 2.3807642459869385,
-      "logits/rejected": 3.3292059898376465,
-      "logps/chosen": -568.1010131835938,
-      "logps/rejected": -696.64453125,
-      "loss": 0.7886,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.7410290241241455,
-      "rewards/margins": 1.2305468320846558,
-      "rewards/rejected": -3.971575975418091,
-      "step": 770
-    },
-    {
-      "epoch": 0.82,
-      "learning_rate": 4.947931323697982e-08,
-      "logits/chosen": 2.637059211730957,
-      "logits/rejected": 3.690800905227661,
-      "logps/chosen": -514.2804565429688,
-      "logps/rejected": -613.6573486328125,
-      "loss": 0.7885,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -2.580803394317627,
-      "rewards/margins": 1.19155752658844,
-      "rewards/rejected": -3.7723612785339355,
-      "step": 780
-    },
-    {
-      "epoch": 0.83,
-      "learning_rate": 4.415420150605398e-08,
-      "logits/chosen": 2.267085313796997,
-      "logits/rejected": 3.2141170501708984,
-      "logps/chosen": -574.6113891601562,
-      "logps/rejected": -717.8250122070312,
-      "loss": 0.7567,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -2.7426769733428955,
-      "rewards/margins": 1.5901119709014893,
-      "rewards/rejected": -4.332788467407227,
-      "step": 790
-    },
-    {
-      "epoch": 0.84,
-      "learning_rate": 3.9104390285376374e-08,
-      "logits/chosen": 1.8068571090698242,
-      "logits/rejected": 3.51857328414917,
-      "logps/chosen": -589.93505859375,
-      "logps/rejected": -680.4630126953125,
-      "loss": 0.7455,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -2.487300395965576,
-      "rewards/margins": 1.4988832473754883,
-      "rewards/rejected": -3.9861836433410645,
-      "step": 800
-    },
-    {
-      "epoch": 0.84,
-      "eval_logits/chosen": 1.9905518293380737,
-      "eval_logits/rejected": 3.243948459625244,
-      "eval_logps/chosen": -526.2843627929688,
-      "eval_logps/rejected": -643.9625854492188,
-      "eval_loss": 0.7792153358459473,
-      "eval_rewards/accuracies": 0.7757936716079712,
-      "eval_rewards/chosen": -2.421745777130127,
-      "eval_rewards/margins": 1.4000948667526245,
-      "eval_rewards/rejected": -3.821840286254883,
-      "eval_runtime": 242.9784,
-      "eval_samples_per_second": 8.231,
-      "eval_steps_per_second": 0.259,
-      "step": 800
-    },
-    {
-      "epoch": 0.85,
-      "learning_rate": 3.433663324986208e-08,
-      "logits/chosen": 2.3890461921691895,
-      "logits/rejected": 3.671307325363159,
-      "logps/chosen": -535.8336791992188,
-      "logps/rejected": -607.342041015625,
-      "loss": 0.7698,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.5636343955993652,
-      "rewards/margins": 1.1194355487823486,
-      "rewards/rejected": -3.683069944381714,
-      "step": 810
-    },
-    {
-      "epoch": 0.86,
-      "learning_rate": 2.9857306851953897e-08,
-      "logits/chosen": 2.8772475719451904,
-      "logits/rejected": 3.245588779449463,
-      "logps/chosen": -489.6429748535156,
-      "logps/rejected": -598.0228271484375,
-      "loss": 0.8105,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.369723081588745,
-      "rewards/margins": 1.2586920261383057,
-      "rewards/rejected": -3.6284148693084717,
-      "step": 820
-    },
-    {
-      "epoch": 0.87,
-      "learning_rate": 2.567240179368185e-08,
-      "logits/chosen": 1.9207391738891602,
-      "logits/rejected": 3.2350220680236816,
-      "logps/chosen": -485.8853454589844,
-      "logps/rejected": -604.6241455078125,
-      "loss": 0.7663,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -2.3702356815338135,
-      "rewards/margins": 1.2538180351257324,
-      "rewards/rejected": -3.624053955078125,
-      "step": 830
-    },
-    {
-      "epoch": 0.88,
-      "learning_rate": 2.1787515014630357e-08,
-      "logits/chosen": 2.4551303386688232,
-      "logits/rejected": 2.7984118461608887,
-      "logps/chosen": -548.1451416015625,
-      "logps/rejected": -633.4719848632812,
-      "loss": 0.8024,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -2.350025177001953,
-      "rewards/margins": 1.1445175409317017,
-      "rewards/rejected": -3.494542360305786,
-      "step": 840
-    },
-    {
-      "epoch": 0.89,
-      "learning_rate": 1.820784220652766e-08,
-      "logits/chosen": 2.1438848972320557,
-      "logits/rejected": 3.26000714302063,
-      "logps/chosen": -524.2879638671875,
-      "logps/rejected": -579.0330200195312,
-      "loss": 0.781,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.2401034832000732,
-      "rewards/margins": 1.1002050638198853,
-      "rewards/rejected": -3.340308666229248,
-      "step": 850
-    },
-    {
-      "epoch": 0.9,
-      "learning_rate": 1.4938170864468636e-08,
-      "logits/chosen": 2.354217290878296,
-      "logits/rejected": 3.4680027961730957,
-      "logps/chosen": -520.9027099609375,
-      "logps/rejected": -637.2523193359375,
-      "loss": 0.7833,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -2.4247303009033203,
-      "rewards/margins": 1.4387235641479492,
-      "rewards/rejected": -3.8634536266326904,
-      "step": 860
-    },
-    {
-      "epoch": 0.91,
-      "learning_rate": 1.1982873884064465e-08,
-      "logits/chosen": 2.0364556312561035,
-      "logits/rejected": 3.0106232166290283,
-      "logps/chosen": -446.37164306640625,
-      "logps/rejected": -599.5112915039062,
-      "loss": 0.7706,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.993843674659729,
-      "rewards/margins": 1.5384770631790161,
-      "rewards/rejected": -3.532320499420166,
-      "step": 870
-    },
-    {
-      "epoch": 0.92,
-      "learning_rate": 9.345903713082304e-09,
-      "logits/chosen": 1.718162178993225,
-      "logits/rejected": 3.142746686935425,
-      "logps/chosen": -520.4351806640625,
-      "logps/rejected": -618.0506591796875,
-      "loss": 0.7503,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.463242769241333,
-      "rewards/margins": 1.1839332580566406,
-      "rewards/rejected": -3.6471760272979736,
-      "step": 880
-    },
-    {
-      "epoch": 0.93,
-      "learning_rate": 7.030787065396865e-09,
-      "logits/chosen": 1.782679796218872,
-      "logits/rejected": 3.1341471672058105,
-      "logps/chosen": -507.89044189453125,
-      "logps/rejected": -622.839111328125,
-      "loss": 0.8135,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -2.4141266345977783,
-      "rewards/margins": 1.1595393419265747,
-      "rewards/rejected": -3.5736660957336426,
-      "step": 890
-    },
-    {
-      "epoch": 0.94,
-      "learning_rate": 5.04062020432286e-09,
-      "logits/chosen": 2.032198190689087,
-      "logits/rejected": 3.270922899246216,
-      "logps/chosen": -533.2232055664062,
-      "logps/rejected": -638.6824340820312,
-      "loss": 0.7785,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -2.348451852798462,
-      "rewards/margins": 1.1957995891571045,
-      "rewards/rejected": -3.5442516803741455,
-      "step": 900
-    },
-    {
-      "epoch": 0.94,
-      "eval_logits/chosen": 1.8721567392349243,
-      "eval_logits/rejected": 3.0941007137298584,
-      "eval_logps/chosen": -513.8541870117188,
-      "eval_logps/rejected": -628.2752685546875,
-      "eval_loss": 0.7778856158256531,
-      "eval_rewards/accuracies": 0.7797619104385376,
-      "eval_rewards/chosen": -2.2974438667297363,
-      "eval_rewards/margins": 1.3675230741500854,
-      "eval_rewards/rejected": -3.6649670600891113,
-      "eval_runtime": 244.1274,
-      "eval_samples_per_second": 8.192,
-      "eval_steps_per_second": 0.258,
-      "step": 900
-    },
-    {
-      "epoch": 0.95,
-      "learning_rate": 3.3780648016376866e-09,
-      "logits/chosen": 2.228888988494873,
-      "logits/rejected": 3.4416561126708984,
-      "logps/chosen": -489.8466796875,
-      "logps/rejected": -613.7962646484375,
-      "loss": 0.7601,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.4230568408966064,
-      "rewards/margins": 1.299936056137085,
-      "rewards/rejected": -3.7229926586151123,
-      "step": 910
-    },
-    {
-      "epoch": 0.96,
-      "learning_rate": 2.0453443778310766e-09,
-      "logits/chosen": 2.0826706886291504,
-      "logits/rejected": 3.181570529937744,
-      "logps/chosen": -532.4862670898438,
-      "logps/rejected": -636.2462158203125,
-      "loss": 0.7257,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -2.3680665493011475,
-      "rewards/margins": 1.4134373664855957,
-      "rewards/rejected": -3.7815041542053223,
-      "step": 920
-    },
-    {
-      "epoch": 0.97,
-      "learning_rate": 1.0442413283435758e-09,
-      "logits/chosen": 1.8148505687713623,
-      "logits/rejected": 2.911557674407959,
-      "logps/chosen": -503.92071533203125,
-      "logps/rejected": -622.6376953125,
-      "loss": 0.7243,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -2.3115131855010986,
-      "rewards/margins": 1.4749082326889038,
-      "rewards/rejected": -3.786421537399292,
-      "step": 930
-    },
     {
       "epoch": 0.98,
-      "learning_rate": 3.760945397705828e-10,
-      "logits/chosen": 2.1273417472839355,
-      "logits/rejected": 3.256913423538208,
-      "logps/chosen": -574.5153198242188,
-      "logps/rejected": -671.7332763671875,
-      "loss": 0.7477,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -2.447880268096924,
-      "rewards/margins": 1.3559364080429077,
-      "rewards/rejected": -3.803816556930542,
-      "step": 940
-    },
-    {
-      "epoch": 0.99,
-      "learning_rate": 4.17975992204056e-11,
-      "logits/chosen": 1.6933950185775757,
-      "logits/rejected": 3.3468966484069824,
-      "logps/chosen": -524.0973510742188,
-      "logps/rejected": -652.6331787109375,
-      "loss": 0.8027,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -2.3957927227020264,
-      "rewards/margins": 1.4984779357910156,
-      "rewards/rejected": -3.894270420074463,
-      "step": 950
+      "eval_logits/chosen": -0.4496035575866699,
+      "eval_logits/rejected": 0.04359949380159378,
+      "eval_logps/chosen": -395.9374084472656,
+      "eval_logps/rejected": -470.5448303222656,
+      "eval_loss": 0.5045374631881714,
+      "eval_rewards/accuracies": 0.726047933101654,
+      "eval_rewards/chosen": -1.182096004486084,
+      "eval_rewards/margins": 0.876003086566925,
+      "eval_rewards/rejected": -2.0580990314483643,
+      "eval_runtime": 494.2334,
+      "eval_samples_per_second": 4.047,
+      "eval_steps_per_second": 0.338,
+      "step": 500
     },
     {
       "epoch": 1.0,
-      "step": 955,
+      "step": 509,
       "total_flos": 0.0,
-      "train_loss": 0.8416462149295507,
-      "train_runtime": 20662.0179,
-      "train_samples_per_second": 2.959,
-      "train_steps_per_second": 0.046
+      "train_loss": 0.5401819272219315,
+      "train_runtime": 34352.758,
+      "train_samples_per_second": 1.78,
+      "train_steps_per_second": 0.015
     }
   ],
   "logging_steps": 10,
-  "max_steps": 955,
+  "max_steps": 509,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
-  "save_steps": 100000000,
+  "save_steps": 1000,
   "total_flos": 0.0,
-  "train_batch_size": 8,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5bee5bf7183364b2486d76255411c15d45960d01e4698754b0948377ea6dc02e
-size 6072
+oid sha256:4326671143f1f88098cb0c8ef537001a10fe9c8346367f54f87f1f4f56ca7627
+size 5944