Model save

README.md

@@ -15,15 +15,15 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 6.9230
-- Rewards/chosen: -4.5175
-- Rewards/rejected: 0.4288
-- Rewards/accuracies: 0.3164
-- Rewards/margins: -4.9464
-- Logps/rejected: -517.5300
-- Logps/chosen: -399.5095
-- Logits/rejected: -4.8908
-- Logits/chosen: -4.6604
+- Loss: 1.9828
+- Rewards/chosen: -3.4223
+- Rewards/rejected: -2.1126
+- Rewards/accuracies: 0.3555
+- Rewards/margins: -1.3097
+- Logps/rejected: -521.2875
+- Logps/chosen: -405.8879
+- Logits/rejected: -4.9364
+- Logits/chosen: -4.7068
 
 ## Model description
 
@@ -42,7 +42,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-07
+- learning_rate: 5e-08
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
@@ -58,16 +58,16 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5909        | 0.2635 | 100  | 6.6534          | -3.0090        | 2.8904           | 0.2773             | -5.8994         | -512.6068      | -396.4924    | -4.8508         | -4.6121       |
-| 0.7239        | 0.5270 | 200  | 8.0720          | -2.8191        | 3.5065           | 0.2734             | -6.3256         | -511.3747      | -396.1127    | -4.9896         | -4.7715       |
-| 0.5556        | 0.7905 | 300  | 6.9230          | -4.5175        | 0.4288           | 0.3164             | -4.9464         | -517.5300      | -399.5095    | -4.8908         | -4.6604       |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.3652        | 0.29  | 100  | 1.7488          | -2.2159        | -1.0097          | 0.3516             | -1.2062         | -519.0817      | -403.4752    | -4.9249         | -4.6962       |
+| 0.312         | 0.57  | 200  | 1.9596          | -3.1949        | -1.8164          | 0.3398             | -1.3786         | -520.6950      | -405.4332    | -4.9391         | -4.7096       |
+| 0.2993        | 0.86  | 300  | 1.9828          | -3.4223        | -2.1126          | 0.3555             | -1.3097         | -521.2875      | -405.8879    | -4.9364         | -4.7068       |
 
 
 ### Framework versions
 
-- Transformers 4.41.1
+- Transformers 4.38.2
 - Pytorch 2.1.2+cu118
 - Datasets 2.16.1
-- Tokenizers 0.19.1
+- Tokenizers 0.15.2

all_results.json

@@ -1,9 +1,8 @@
 {
-    "epoch": 0.9986824769433466,
-    "total_flos": 0.0,
-    "train_loss": 0.5517442987587962,
-    "train_runtime": 6181.8185,
-    "train_samples": 48530,
-    "train_samples_per_second": 7.85,
-    "train_steps_per_second": 0.061
+    "epoch": 1.0,
+    "train_loss": 0.3696309270011661,
+    "train_runtime": 5591.2149,
+    "train_samples": 44682,
+    "train_samples_per_second": 7.991,
+    "train_steps_per_second": 0.062
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.41.1",
+  "transformers_version": "4.38.2",
   "use_cache": false,
   "vocab_size": 32002
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
-  "transformers_version": "4.41.1"
+  "transformers_version": "4.38.2"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fad6798eb6bb6153f6b5aff19d3bc5cffa1aeb40bf26f57d5a587788a415d08c
+oid sha256:b289b436dc9787cb73fff5a2a0c5fb69ad78dc06d960a4946cb9d6b35bf920b3
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20ca669573e568702fa6d40967a3d29846dc5ecf876326abd24dbc1904e69161
+oid sha256:556d3f80045da81e8b580005d0676f922550a32c41b802ac16391cdb8a56eac7
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ddf6a93711a98049fe876e8b35960fb84be7446726bf529c7236bf8c1e7f0d9d
+oid sha256:06047de541b6531a598ee5f5cc63aaa960471d4a4c6da134647cc69aa23f71b0
 size 4540532728

runs/May29_23-11-40_n136-129-074/events.out.tfevents.1716995522.n136-129-074.2191665.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:306ad423168f6c7a36b9bc75ec55c766a7028a9abbdc8ffebb6e790aa8f8b678
-size 28284
+oid sha256:b7e85772f2279a93f5a7a61be83422e97cc2aacc7c45c6008ebf29def9ccc889
+size 31390

tokenizer.json

@@ -152,7 +152,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,9 +1,8 @@
 {
-    "epoch": 0.9986824769433466,
-    "total_flos": 0.0,
-    "train_loss": 0.5517442987587962,
-    "train_runtime": 6181.8185,
-    "train_samples": 48530,
-    "train_samples_per_second": 7.85,
-    "train_steps_per_second": 0.061
+    "epoch": 1.0,
+    "train_loss": 0.3696309270011661,
+    "train_runtime": 5591.2149,
+    "train_samples": 44682,
+    "train_samples_per_second": 7.991,
+    "train_steps_per_second": 0.062
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9986824769433466,
+  "epoch": 0.9985693848354793,
   "eval_steps": 100,
-  "global_step": 379,
+  "global_step": 349,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.002635046113306983,
-      "grad_norm": 1701.3284378032004,
-      "learning_rate": 1.3157894736842104e-08,
-      "logits/chosen": -4.685327529907227,
-      "logits/rejected": -4.87608528137207,
-      "logps/chosen": -207.7137451171875,
-      "logps/rejected": -145.5098114013672,
+      "epoch": 0.0,
+      "grad_norm": 1482.8898632632786,
+      "learning_rate": 1.4285714285714284e-09,
+      "logits/chosen": -4.490396976470947,
+      "logits/rejected": -4.787891387939453,
+      "logps/chosen": -300.56573486328125,
+      "logps/rejected": -263.39849853515625,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,635 +24,578 @@
       "step": 1
     },
     {
-      "epoch": 0.026350461133069828,
-      "grad_norm": 1584.9120962726856,
-      "learning_rate": 1.3157894736842104e-07,
-      "logits/chosen": -4.499300479888916,
-      "logits/rejected": -4.840802192687988,
-      "logps/chosen": -223.6631317138672,
-      "logps/rejected": -160.81097412109375,
-      "loss": 0.7136,
-      "rewards/accuracies": 0.4444444477558136,
-      "rewards/chosen": 0.004151582717895508,
-      "rewards/margins": 0.002958830911666155,
-      "rewards/rejected": 0.001192751806229353,
+      "epoch": 0.03,
+      "grad_norm": 1626.423546399318,
+      "learning_rate": 1.4285714285714284e-08,
+      "logits/chosen": -4.29224967956543,
+      "logits/rejected": -4.469963550567627,
+      "logps/chosen": -285.7841796875,
+      "logps/rejected": -241.37472534179688,
+      "loss": 0.7238,
+      "rewards/accuracies": 0.4097222089767456,
+      "rewards/chosen": 0.00654969597235322,
+      "rewards/margins": -0.022695984691381454,
+      "rewards/rejected": 0.029245682060718536,
       "step": 10
     },
     {
-      "epoch": 0.052700922266139656,
-      "grad_norm": 955.29292445595,
-      "learning_rate": 2.631578947368421e-07,
-      "logits/chosen": -4.518028259277344,
-      "logits/rejected": -4.817793846130371,
-      "logps/chosen": -220.1512451171875,
-      "logps/rejected": -172.69322204589844,
-      "loss": 0.4939,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.9042474627494812,
-      "rewards/margins": 0.9315685033798218,
-      "rewards/rejected": -0.02732105180621147,
+      "epoch": 0.06,
+      "grad_norm": 1614.6738239528886,
+      "learning_rate": 2.857142857142857e-08,
+      "logits/chosen": -4.210062026977539,
+      "logits/rejected": -4.502069473266602,
+      "logps/chosen": -312.73968505859375,
+      "logps/rejected": -247.76315307617188,
+      "loss": 0.7059,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.01807965151965618,
+      "rewards/margins": 0.015493685379624367,
+      "rewards/rejected": 0.0025859654415398836,
       "step": 20
     },
     {
-      "epoch": 0.07905138339920949,
-      "grad_norm": 1078.6380127502835,
-      "learning_rate": 3.9473684210526315e-07,
-      "logits/chosen": -4.58280086517334,
-      "logits/rejected": -4.878857612609863,
-      "logps/chosen": -212.6325225830078,
-      "logps/rejected": -183.79238891601562,
-      "loss": 0.3906,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.010448932647705,
-      "rewards/margins": 2.6640021800994873,
-      "rewards/rejected": 0.346446692943573,
+      "epoch": 0.09,
+      "grad_norm": 1444.832849324016,
+      "learning_rate": 4.285714285714285e-08,
+      "logits/chosen": -4.140606880187988,
+      "logits/rejected": -4.359016418457031,
+      "logps/chosen": -323.00701904296875,
+      "logps/rejected": -274.5583801269531,
+      "loss": 0.6754,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": 0.26756447553634644,
+      "rewards/margins": 0.19910377264022827,
+      "rewards/rejected": 0.06846068799495697,
       "step": 30
     },
     {
-      "epoch": 0.10540184453227931,
-      "grad_norm": 1034.6942446334156,
-      "learning_rate": 4.999575626062319e-07,
-      "logits/chosen": -4.496463298797607,
-      "logits/rejected": -4.832797050476074,
-      "logps/chosen": -225.2804412841797,
-      "logps/rejected": -177.3509063720703,
-      "loss": 0.4804,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 4.241217136383057,
-      "rewards/margins": 4.989673614501953,
-      "rewards/rejected": -0.7484563589096069,
+      "epoch": 0.11,
+      "grad_norm": 1322.1199574632033,
+      "learning_rate": 4.9968724814144246e-08,
+      "logits/chosen": -4.296597480773926,
+      "logits/rejected": -4.558178901672363,
+      "logps/chosen": -277.67041015625,
+      "logps/rejected": -227.9638214111328,
+      "loss": 0.5887,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.46597638726234436,
+      "rewards/margins": 0.30959779024124146,
+      "rewards/rejected": 0.1563786268234253,
       "step": 40
     },
     {
-      "epoch": 0.13175230566534915,
-      "grad_norm": 1292.8962171873402,
-      "learning_rate": 4.984737660598186e-07,
-      "logits/chosen": -4.517908573150635,
-      "logits/rejected": -4.786294937133789,
-      "logps/chosen": -214.01718139648438,
-      "logps/rejected": -174.09678649902344,
-      "loss": 0.5163,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 3.7782645225524902,
-      "rewards/margins": 5.338944435119629,
-      "rewards/rejected": -1.5606796741485596,
+      "epoch": 0.14,
+      "grad_norm": 1226.7969075975607,
+      "learning_rate": 4.971899263739325e-08,
+      "logits/chosen": -4.339926719665527,
+      "logits/rejected": -4.592724323272705,
+      "logps/chosen": -270.9559020996094,
+      "logps/rejected": -228.2990264892578,
+      "loss": 0.5099,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": 0.9494625329971313,
+      "rewards/margins": 0.5203765034675598,
+      "rewards/rejected": 0.4290861189365387,
       "step": 50
     },
     {
-      "epoch": 0.15810276679841898,
-      "grad_norm": 1081.6576652859876,
-      "learning_rate": 4.948824853131236e-07,
-      "logits/chosen": -4.719171047210693,
-      "logits/rejected": -4.9818878173828125,
-      "logps/chosen": -215.00869750976562,
-      "logps/rejected": -180.1785430908203,
-      "loss": 0.4821,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 3.4492969512939453,
-      "rewards/margins": 6.232473373413086,
-      "rewards/rejected": -2.7831759452819824,
+      "epoch": 0.17,
+      "grad_norm": 929.2104534934765,
+      "learning_rate": 4.9222026055025726e-08,
+      "logits/chosen": -4.333067893981934,
+      "logits/rejected": -4.564365386962891,
+      "logps/chosen": -303.43310546875,
+      "logps/rejected": -252.1361846923828,
+      "loss": 0.4317,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 1.6878736019134521,
+      "rewards/margins": 1.0692347288131714,
+      "rewards/rejected": 0.6186389923095703,
       "step": 60
     },
     {
-      "epoch": 0.1844532279314888,
-      "grad_norm": 780.7579097796144,
-      "learning_rate": 4.892141805936084e-07,
-      "logits/chosen": -4.691411018371582,
-      "logits/rejected": -4.9814910888671875,
-      "logps/chosen": -222.72189331054688,
-      "logps/rejected": -186.29788208007812,
-      "loss": 0.5005,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 3.7675652503967285,
-      "rewards/margins": 6.836648464202881,
-      "rewards/rejected": -3.0690836906433105,
+      "epoch": 0.2,
+      "grad_norm": 1085.9024365118285,
+      "learning_rate": 4.8482795627104736e-08,
+      "logits/chosen": -4.332296371459961,
+      "logits/rejected": -4.567011833190918,
+      "logps/chosen": -289.8699645996094,
+      "logps/rejected": -241.077880859375,
+      "loss": 0.4089,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 1.8221668004989624,
+      "rewards/margins": 1.204461693763733,
+      "rewards/rejected": 0.6177049875259399,
       "step": 70
     },
     {
-      "epoch": 0.21080368906455862,
-      "grad_norm": 1751.5015463679365,
-      "learning_rate": 4.81516928858564e-07,
-      "logits/chosen": -4.691437721252441,
-      "logits/rejected": -4.943267822265625,
-      "logps/chosen": -219.7614288330078,
-      "logps/rejected": -179.60899353027344,
-      "loss": 0.5615,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 3.4195969104766846,
-      "rewards/margins": 6.783135890960693,
-      "rewards/rejected": -3.363539457321167,
+      "epoch": 0.23,
+      "grad_norm": 978.9007166471121,
+      "learning_rate": 4.750869498807735e-08,
+      "logits/chosen": -4.28688383102417,
+      "logits/rejected": -4.543013572692871,
+      "logps/chosen": -322.56048583984375,
+      "logps/rejected": -271.0233154296875,
+      "loss": 0.3625,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.1792995929718018,
+      "rewards/margins": 1.4446518421173096,
+      "rewards/rejected": 0.7346473932266235,
       "step": 80
     },
     {
-      "epoch": 0.23715415019762845,
-      "grad_norm": 1233.3333710128843,
-      "learning_rate": 4.7185601601995784e-07,
-      "logits/chosen": -4.569981575012207,
-      "logits/rejected": -4.954944610595703,
-      "logps/chosen": -212.4117431640625,
-      "logps/rejected": -173.86911010742188,
-      "loss": 0.4989,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 3.7934730052948,
-      "rewards/margins": 7.21230936050415,
-      "rewards/rejected": -3.4188361167907715,
+      "epoch": 0.26,
+      "grad_norm": 755.2675019751182,
+      "learning_rate": 4.630946689712609e-08,
+      "logits/chosen": -4.393240928649902,
+      "logits/rejected": -4.636073112487793,
+      "logps/chosen": -283.79290771484375,
+      "logps/rejected": -234.5099334716797,
+      "loss": 0.3436,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 2.397665500640869,
+      "rewards/margins": 1.638594388961792,
+      "rewards/rejected": 0.7590711116790771,
       "step": 90
     },
     {
-      "epoch": 0.2635046113306983,
-      "grad_norm": 1244.999056136522,
-      "learning_rate": 4.603133832077953e-07,
-      "logits/chosen": -4.672577381134033,
-      "logits/rejected": -4.9231157302856445,
-      "logps/chosen": -206.11221313476562,
-      "logps/rejected": -176.262451171875,
-      "loss": 0.5909,
+      "epoch": 0.29,
+      "grad_norm": 893.2972482207422,
+      "learning_rate": 4.4897105793046024e-08,
+      "logits/chosen": -4.302220344543457,
+      "logits/rejected": -4.582569122314453,
+      "logps/chosen": -270.37457275390625,
+      "logps/rejected": -229.5946502685547,
+      "loss": 0.3652,
       "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.6933536529541016,
-      "rewards/margins": 6.304307460784912,
-      "rewards/rejected": -2.6109542846679688,
+      "rewards/chosen": 2.673680067062378,
+      "rewards/margins": 2.020587921142578,
+      "rewards/rejected": 0.6530919075012207,
       "step": 100
     },
     {
-      "epoch": 0.2635046113306983,
-      "eval_logits/chosen": -4.612120628356934,
-      "eval_logits/rejected": -4.85081672668457,
-      "eval_logps/chosen": -396.4924011230469,
-      "eval_logps/rejected": -512.6068115234375,
-      "eval_loss": 6.653407096862793,
-      "eval_rewards/accuracies": 0.27734375,
-      "eval_rewards/chosen": -3.0089728832244873,
-      "eval_rewards/margins": -5.8993988037109375,
-      "eval_rewards/rejected": 2.890425682067871,
-      "eval_runtime": 98.318,
-      "eval_samples_per_second": 20.342,
-      "eval_steps_per_second": 0.325,
+      "epoch": 0.29,
+      "eval_logits/chosen": -4.696198463439941,
+      "eval_logits/rejected": -4.924900054931641,
+      "eval_logps/chosen": -403.4751892089844,
+      "eval_logps/rejected": -519.0816650390625,
+      "eval_loss": 1.7488452196121216,
+      "eval_rewards/accuracies": 0.3515625,
+      "eval_rewards/chosen": -2.2159295082092285,
+      "eval_rewards/margins": -1.206247329711914,
+      "eval_rewards/rejected": -1.009682059288025,
+      "eval_runtime": 97.4797,
+      "eval_samples_per_second": 20.517,
+      "eval_steps_per_second": 0.328,
       "step": 100
     },
     {
-      "epoch": 0.2898550724637681,
-      "grad_norm": 870.1595065644425,
-      "learning_rate": 4.4698693176863316e-07,
-      "logits/chosen": -4.71740198135376,
-      "logits/rejected": -4.952963829040527,
-      "logps/chosen": -208.0568084716797,
-      "logps/rejected": -172.92605590820312,
-      "loss": 0.4837,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.114009380340576,
-      "rewards/margins": 7.352834224700928,
-      "rewards/rejected": -4.238823890686035,
+      "epoch": 0.31,
+      "grad_norm": 872.5882367590035,
+      "learning_rate": 4.328573782827409e-08,
+      "logits/chosen": -4.3329758644104,
+      "logits/rejected": -4.617272853851318,
+      "logps/chosen": -296.02618408203125,
+      "logps/rejected": -242.73428344726562,
+      "loss": 0.349,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.7564496994018555,
+      "rewards/margins": 2.076282024383545,
+      "rewards/rejected": 0.6801677942276001,
       "step": 110
     },
     {
-      "epoch": 0.31620553359683795,
-      "grad_norm": 1230.5660125025618,
-      "learning_rate": 4.319896928940505e-07,
-      "logits/chosen": -4.8249382972717285,
-      "logits/rejected": -5.139795780181885,
-      "logps/chosen": -204.9242401123047,
-      "logps/rejected": -181.16221618652344,
-      "loss": 0.6657,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 4.0227861404418945,
-      "rewards/margins": 6.973240852355957,
-      "rewards/rejected": -2.950455904006958,
+      "epoch": 0.34,
+      "grad_norm": 1043.3393138533181,
+      "learning_rate": 4.1491479581946166e-08,
+      "logits/chosen": -4.3465352058410645,
+      "logits/rejected": -4.575117111206055,
+      "logps/chosen": -300.99859619140625,
+      "logps/rejected": -265.1387023925781,
+      "loss": 0.3567,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 2.6209876537323,
+      "rewards/margins": 1.951716423034668,
+      "rewards/rejected": 0.6692714095115662,
       "step": 120
     },
     {
-      "epoch": 0.3425559947299078,
-      "grad_norm": 1400.2090737238004,
-      "learning_rate": 4.1544886892205354e-07,
-      "logits/chosen": -4.680369853973389,
-      "logits/rejected": -4.979363441467285,
-      "logps/chosen": -218.03897094726562,
-      "logps/rejected": -192.44757080078125,
-      "loss": 0.4732,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 3.1923911571502686,
-      "rewards/margins": 8.124491691589355,
-      "rewards/rejected": -4.932101249694824,
+      "epoch": 0.37,
+      "grad_norm": 837.2392359904699,
+      "learning_rate": 3.953227686510564e-08,
+      "logits/chosen": -4.319238185882568,
+      "logits/rejected": -4.565236568450928,
+      "logps/chosen": -288.36688232421875,
+      "logps/rejected": -240.29177856445312,
+      "loss": 0.3089,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 2.86527419090271,
+      "rewards/margins": 2.168524980545044,
+      "rewards/rejected": 0.6967490911483765,
       "step": 130
     },
     {
-      "epoch": 0.3689064558629776,
-      "grad_norm": 1284.6303869923338,
-      "learning_rate": 3.975047544428254e-07,
-      "logits/chosen": -4.738985061645508,
-      "logits/rejected": -4.995828628540039,
-      "logps/chosen": -200.0480499267578,
-      "logps/rejected": -173.74069213867188,
-      "loss": 0.7249,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 3.118265151977539,
-      "rewards/margins": 6.879487037658691,
-      "rewards/rejected": -3.7612221240997314,
+      "epoch": 0.4,
+      "grad_norm": 1077.8316620275032,
+      "learning_rate": 3.7427725230301354e-08,
+      "logits/chosen": -4.310965538024902,
+      "logits/rejected": -4.550887584686279,
+      "logps/chosen": -295.83868408203125,
+      "logps/rejected": -240.91311645507812,
+      "loss": 0.3113,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 3.000100612640381,
+      "rewards/margins": 2.261690855026245,
+      "rewards/rejected": 0.7384099960327148,
       "step": 140
     },
     {
-      "epoch": 0.3952569169960474,
-      "grad_norm": 1087.9280552765,
-      "learning_rate": 3.78309546359696e-07,
-      "logits/chosen": -4.816591739654541,
-      "logits/rejected": -5.046236991882324,
-      "logps/chosen": -197.75784301757812,
-      "logps/rejected": -187.12631225585938,
-      "loss": 0.6191,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 1.7183030843734741,
-      "rewards/margins": 8.374044418334961,
-      "rewards/rejected": -6.655740261077881,
+      "epoch": 0.43,
+      "grad_norm": 805.166891537404,
+      "learning_rate": 3.5198873980801955e-08,
+      "logits/chosen": -4.3193488121032715,
+      "logits/rejected": -4.625092506408691,
+      "logps/chosen": -294.2840881347656,
+      "logps/rejected": -245.18655395507812,
+      "loss": 0.318,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 3.1946704387664795,
+      "rewards/margins": 2.1761879920959473,
+      "rewards/rejected": 1.0184824466705322,
       "step": 150
     },
     {
-      "epoch": 0.42160737812911725,
-      "grad_norm": 870.0436658351372,
-      "learning_rate": 3.580260529980584e-07,
-      "logits/chosen": -4.622679233551025,
-      "logits/rejected": -4.90102481842041,
-      "logps/chosen": -210.0752410888672,
-      "logps/rejected": -182.36378479003906,
-      "loss": 0.5743,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 2.5026469230651855,
-      "rewards/margins": 6.609257698059082,
-      "rewards/rejected": -4.106611251831055,
+      "epoch": 0.46,
+      "grad_norm": 711.9582620759091,
+      "learning_rate": 3.2868015639687205e-08,
+      "logits/chosen": -4.259413242340088,
+      "logits/rejected": -4.450949668884277,
+      "logps/chosen": -311.91326904296875,
+      "logps/rejected": -270.4508972167969,
+      "loss": 0.3344,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 3.2649455070495605,
+      "rewards/margins": 2.298511266708374,
+      "rewards/rejected": 0.966434121131897,
       "step": 160
     },
     {
-      "epoch": 0.4479578392621871,
-      "grad_norm": 1109.7545725706777,
-      "learning_rate": 3.36826313211205e-07,
-      "logits/chosen": -4.862623691558838,
-      "logits/rejected": -5.1108198165893555,
-      "logps/chosen": -195.3279266357422,
-      "logps/rejected": -170.62802124023438,
-      "loss": 0.6483,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 2.5267701148986816,
-      "rewards/margins": 7.295065402984619,
-      "rewards/rejected": -4.768294811248779,
+      "epoch": 0.49,
+      "grad_norm": 894.588925085696,
+      "learning_rate": 3.0458462984504134e-08,
+      "logits/chosen": -4.303664207458496,
+      "logits/rejected": -4.500936031341553,
+      "logps/chosen": -274.3706359863281,
+      "logps/rejected": -235.48678588867188,
+      "loss": 0.2963,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 3.1083619594573975,
+      "rewards/margins": 2.440159559249878,
+      "rewards/rejected": 0.6682023406028748,
       "step": 170
     },
     {
-      "epoch": 0.4743083003952569,
-      "grad_norm": 1174.4955529247104,
-      "learning_rate": 3.14890137195437e-07,
-      "logits/chosen": -4.831478595733643,
-      "logits/rejected": -5.037031173706055,
-      "logps/chosen": -211.2603302001953,
-      "logps/rejected": -190.32931518554688,
-      "loss": 0.5829,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 1.8433873653411865,
-      "rewards/margins": 7.003817558288574,
-      "rewards/rejected": -5.160429954528809,
+      "epoch": 0.52,
+      "grad_norm": 787.4594621489149,
+      "learning_rate": 2.7994315877542628e-08,
+      "logits/chosen": -4.386083602905273,
+      "logits/rejected": -4.545100212097168,
+      "logps/chosen": -263.3323059082031,
+      "logps/rejected": -228.91329956054688,
+      "loss": 0.329,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 2.6448895931243896,
+      "rewards/margins": 2.054766893386841,
+      "rewards/rejected": 0.5901231169700623,
       "step": 180
     },
     {
-      "epoch": 0.5006587615283268,
-      "grad_norm": 1405.2874598551575,
-      "learning_rate": 2.9240358139084013e-07,
-      "logits/chosen": -4.9419050216674805,
-      "logits/rejected": -5.200289726257324,
-      "logps/chosen": -209.5860595703125,
-      "logps/rejected": -188.95069885253906,
-      "loss": 0.5485,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 1.218836784362793,
-      "rewards/margins": 8.241748809814453,
-      "rewards/rejected": -7.022912502288818,
+      "epoch": 0.54,
+      "grad_norm": 1024.0437070924656,
+      "learning_rate": 2.5500220223847356e-08,
+      "logits/chosen": -4.346238613128662,
+      "logits/rejected": -4.559948444366455,
+      "logps/chosen": -290.6450500488281,
+      "logps/rejected": -252.0496063232422,
+      "loss": 0.3385,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 3.277946949005127,
+      "rewards/margins": 2.545849323272705,
+      "rewards/rejected": 0.7320975661277771,
       "step": 190
     },
     {
-      "epoch": 0.5270092226613966,
-      "grad_norm": 1006.939051019727,
-      "learning_rate": 2.695573704031885e-07,
-      "logits/chosen": -4.7473955154418945,
-      "logits/rejected": -5.000621795654297,
-      "logps/chosen": -217.6959686279297,
-      "logps/rejected": -190.3670196533203,
-      "loss": 0.7239,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.8086190223693848,
-      "rewards/margins": 6.961747646331787,
-      "rewards/rejected": -3.1531288623809814,
+      "epoch": 0.57,
+      "grad_norm": 937.2977444102543,
+      "learning_rate": 2.3001121467819626e-08,
+      "logits/chosen": -4.290076732635498,
+      "logits/rejected": -4.555208683013916,
+      "logps/chosen": -317.6783447265625,
+      "logps/rejected": -267.9818115234375,
+      "loss": 0.312,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 3.4544365406036377,
+      "rewards/margins": 2.3504996299743652,
+      "rewards/rejected": 1.1039369106292725,
       "step": 200
     },
     {
-      "epoch": 0.5270092226613966,
-      "eval_logits/chosen": -4.771502494812012,
-      "eval_logits/rejected": -4.989596843719482,
-      "eval_logps/chosen": -396.1126708984375,
-      "eval_logps/rejected": -511.37469482421875,
-      "eval_loss": 8.072031021118164,
-      "eval_rewards/accuracies": 0.2734375,
-      "eval_rewards/chosen": -2.8191022872924805,
-      "eval_rewards/margins": -6.325590133666992,
-      "eval_rewards/rejected": 3.5064878463745117,
-      "eval_runtime": 97.9327,
-      "eval_samples_per_second": 20.422,
-      "eval_steps_per_second": 0.327,
+      "epoch": 0.57,
+      "eval_logits/chosen": -4.7096052169799805,
+      "eval_logits/rejected": -4.93910026550293,
+      "eval_logps/chosen": -405.4331970214844,
+      "eval_logps/rejected": -520.6950073242188,
+      "eval_loss": 1.959585189819336,
+      "eval_rewards/accuracies": 0.33984375,
+      "eval_rewards/chosen": -3.194929599761963,
+      "eval_rewards/margins": -1.3785794973373413,
+      "eval_rewards/rejected": -1.8163501024246216,
+      "eval_runtime": 97.467,
+      "eval_samples_per_second": 20.52,
+      "eval_steps_per_second": 0.328,
       "step": 200
     },
     {
-      "epoch": 0.5533596837944664,
-      "grad_norm": 1103.8948285514384,
-      "learning_rate": 2.465452793317865e-07,
-      "logits/chosen": -4.7866034507751465,
-      "logits/rejected": -5.081458568572998,
-      "logps/chosen": -228.73922729492188,
-      "logps/rejected": -201.91409301757812,
-      "loss": 0.5681,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 2.7976126670837402,
-      "rewards/margins": 7.8736419677734375,
-      "rewards/rejected": -5.076028823852539,
+      "epoch": 0.6,
+      "grad_norm": 839.0813036142636,
+      "learning_rate": 2.0522015093886612e-08,
+      "logits/chosen": -4.317226409912109,
+      "logits/rejected": -4.618459224700928,
+      "logps/chosen": -294.03253173828125,
+      "logps/rejected": -234.196533203125,
+      "loss": 0.2938,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 3.499882459640503,
+      "rewards/margins": 2.7310853004455566,
+      "rewards/rejected": 0.7687975168228149,
       "step": 210
     },
     {
-      "epoch": 0.5797101449275363,
-      "grad_norm": 715.6592810091468,
-      "learning_rate": 2.2356249022388789e-07,
-      "logits/chosen": -4.758913516998291,
-      "logits/rejected": -5.04398250579834,
-      "logps/chosen": -205.92568969726562,
-      "logps/rejected": -170.3586883544922,
-      "loss": 0.4895,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 4.249292373657227,
-      "rewards/margins": 7.541260719299316,
-      "rewards/rejected": -3.291968822479248,
+      "epoch": 0.63,
+      "grad_norm": 996.0408528817461,
+      "learning_rate": 1.808769662668035e-08,
+      "logits/chosen": -4.358494758605957,
+      "logits/rejected": -4.704705238342285,
+      "logps/chosen": -299.7410888671875,
+      "logps/rejected": -235.092041015625,
+      "loss": 0.2943,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 3.0801563262939453,
+      "rewards/margins": 2.7920236587524414,
+      "rewards/rejected": 0.28813308477401733,
       "step": 220
     },
     {
-      "epoch": 0.6060606060606061,
-      "grad_norm": 944.8747719910025,
-      "learning_rate": 2.0080393659578038e-07,
-      "logits/chosen": -4.764594078063965,
-      "logits/rejected": -5.100175857543945,
-      "logps/chosen": -210.0486297607422,
-      "logps/rejected": -179.91928100585938,
-      "loss": 0.5197,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 5.489472389221191,
-      "rewards/margins": 8.828967094421387,
-      "rewards/rejected": -3.3394947052001953,
+      "epoch": 0.66,
+      "grad_norm": 781.9269603792492,
+      "learning_rate": 1.5722513631174444e-08,
+      "logits/chosen": -4.392434597015381,
+      "logits/rejected": -4.748046398162842,
+      "logps/chosen": -289.19927978515625,
+      "logps/rejected": -234.593017578125,
+      "loss": 0.3084,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 2.994288921356201,
+      "rewards/margins": 2.508836507797241,
+      "rewards/rejected": 0.4854525625705719,
       "step": 230
     },
     {
-      "epoch": 0.6324110671936759,
-      "grad_norm": 1267.6123006349214,
-      "learning_rate": 1.7846265006183976e-07,
-      "logits/chosen": -4.821089744567871,
-      "logits/rejected": -5.0871124267578125,
-      "logps/chosen": -211.7589874267578,
-      "logps/rejected": -181.33819580078125,
-      "loss": 0.5444,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -0.23131565749645233,
-      "rewards/margins": 6.660643577575684,
-      "rewards/rejected": -6.891959190368652,
+      "epoch": 0.69,
+      "grad_norm": 795.1407214961006,
+      "learning_rate": 1.345012219322345e-08,
+      "logits/chosen": -4.238420486450195,
+      "logits/rejected": -4.548759460449219,
+      "logps/chosen": -285.19232177734375,
+      "logps/rejected": -240.72509765625,
+      "loss": 0.3269,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.8941564559936523,
+      "rewards/margins": 2.3365750312805176,
+      "rewards/rejected": 0.5575811266899109,
       "step": 240
     },
     {
-      "epoch": 0.6587615283267457,
-      "grad_norm": 911.8592996107284,
-      "learning_rate": 1.5672812309497722e-07,
-      "logits/chosen": -4.781493663787842,
-      "logits/rejected": -5.052074432373047,
-      "logps/chosen": -201.82290649414062,
-      "logps/rejected": -175.06719970703125,
-      "loss": 0.5887,
+      "epoch": 0.72,
+      "grad_norm": 879.951904317175,
+      "learning_rate": 1.1293250316137664e-08,
+      "logits/chosen": -4.293431282043457,
+      "logits/rejected": -4.5221476554870605,
+      "logps/chosen": -294.2128601074219,
+      "logps/rejected": -254.7281494140625,
+      "loss": 0.3131,
       "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 2.856235980987549,
-      "rewards/margins": 8.613394737243652,
-      "rewards/rejected": -5.757159233093262,
+      "rewards/chosen": 3.089404821395874,
+      "rewards/margins": 2.4765713214874268,
+      "rewards/rejected": 0.6128337383270264,
       "step": 250
     },
     {
-      "epoch": 0.6851119894598156,
-      "grad_norm": 840.8020636445922,
-      "learning_rate": 1.357847018050843e-07,
-      "logits/chosen": -4.674568176269531,
-      "logits/rejected": -4.961749076843262,
-      "logps/chosen": -232.63119506835938,
-      "logps/rejected": -204.2981719970703,
-      "loss": 0.6421,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": 2.906367778778076,
-      "rewards/margins": 7.688606262207031,
-      "rewards/rejected": -4.782238483428955,
+      "epoch": 0.74,
+      "grad_norm": 1094.0422336259135,
+      "learning_rate": 9.273470599753375e-09,
+      "logits/chosen": -4.372658729553223,
+      "logits/rejected": -4.525150775909424,
+      "logps/chosen": -294.89874267578125,
+      "logps/rejected": -263.60577392578125,
+      "loss": 0.3409,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 2.7965197563171387,
+      "rewards/margins": 2.07285737991333,
+      "rewards/rejected": 0.7236624956130981,
       "step": 260
     },
     {
-      "epoch": 0.7114624505928854,
-      "grad_norm": 787.2425305884327,
-      "learning_rate": 1.1581002236747328e-07,
-      "logits/chosen": -4.637770652770996,
-      "logits/rejected": -4.975251197814941,
-      "logps/chosen": -190.36253356933594,
-      "logps/rejected": -166.21788024902344,
-      "loss": 0.5113,
+      "epoch": 0.77,
+      "grad_norm": 775.0184329679462,
+      "learning_rate": 7.410984475616819e-09,
+      "logits/chosen": -4.244694709777832,
+      "logits/rejected": -4.5279622077941895,
+      "logps/chosen": -282.0103454589844,
+      "logps/rejected": -239.31399536132812,
+      "loss": 0.2834,
       "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 2.2253878116607666,
-      "rewards/margins": 8.868097305297852,
-      "rewards/rejected": -6.6427106857299805,
+      "rewards/chosen": 3.1753017902374268,
+      "rewards/margins": 2.7872982025146484,
+      "rewards/rejected": 0.38800328969955444,
       "step": 270
     },
     {
-      "epoch": 0.7378129117259552,
-      "grad_norm": 1064.0700290859438,
-      "learning_rate": 9.697350436308427e-08,
-      "logits/chosen": -4.585692405700684,
-      "logits/rejected": -4.871885776519775,
-      "logps/chosen": -229.0143585205078,
-      "logps/rejected": -194.08761596679688,
-      "loss": 0.5252,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.0419421195983887,
-      "rewards/margins": 8.473891258239746,
-      "rewards/rejected": -5.431949138641357,
+      "epoch": 0.8,
+      "grad_norm": 796.3667611732337,
+      "learning_rate": 5.724420156318405e-09,
+      "logits/chosen": -4.291975498199463,
+      "logits/rejected": -4.621099472045898,
+      "logps/chosen": -285.0769958496094,
+      "logps/rejected": -232.0653839111328,
+      "loss": 0.3144,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.847846508026123,
+      "rewards/margins": 2.3038220405578613,
+      "rewards/rejected": 0.5440241098403931,
       "step": 280
     },
     {
-      "epoch": 0.764163372859025,
-      "grad_norm": 839.1911415542294,
-      "learning_rate": 7.943491380952188e-08,
-      "logits/chosen": -4.828024864196777,
-      "logits/rejected": -5.060397148132324,
-      "logps/chosen": -196.42611694335938,
-      "logps/rejected": -171.01446533203125,
-      "loss": 0.4822,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 4.435191631317139,
-      "rewards/margins": 8.396050453186035,
-      "rewards/rejected": -3.9608588218688965,
+      "epoch": 0.83,
+      "grad_norm": 1101.8200803185337,
+      "learning_rate": 4.230646319847259e-09,
+      "logits/chosen": -4.424475193023682,
+      "logits/rejected": -4.705140113830566,
+      "logps/chosen": -295.53857421875,
+      "logps/rejected": -232.05245971679688,
+      "loss": 0.3121,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.7051048278808594,
+      "rewards/margins": 2.4193003177642822,
+      "rewards/rejected": 0.2858046591281891,
       "step": 290
     },
     {
-      "epoch": 0.7905138339920948,
-      "grad_norm": 887.4606387846492,
-      "learning_rate": 6.334300807088508e-08,
-      "logits/chosen": -4.608688831329346,
-      "logits/rejected": -4.9593987464904785,
-      "logps/chosen": -195.76962280273438,
-      "logps/rejected": -164.97048950195312,
-      "loss": 0.5556,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 3.875316619873047,
-      "rewards/margins": 8.369561195373535,
-      "rewards/rejected": -4.4942450523376465,
+      "epoch": 0.86,
+      "grad_norm": 1165.1379404537536,
+      "learning_rate": 2.944603392457931e-09,
+      "logits/chosen": -4.335474491119385,
+      "logits/rejected": -4.49271297454834,
+      "logps/chosen": -292.6704406738281,
+      "logps/rejected": -252.3407440185547,
+      "loss": 0.2993,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 3.1620662212371826,
+      "rewards/margins": 2.2422008514404297,
+      "rewards/rejected": 0.9198653101921082,
       "step": 300
     },
     {
-      "epoch": 0.7905138339920948,
-      "eval_logits/chosen": -4.660388469696045,
-      "eval_logits/rejected": -4.890798091888428,
-      "eval_logps/chosen": -399.5094909667969,
-      "eval_logps/rejected": -517.530029296875,
-      "eval_loss": 6.923000335693359,
-      "eval_rewards/accuracies": 0.31640625,
-      "eval_rewards/chosen": -4.517519474029541,
-      "eval_rewards/margins": -4.946353435516357,
-      "eval_rewards/rejected": 0.42883408069610596,
-      "eval_runtime": 99.1917,
-      "eval_samples_per_second": 20.163,
-      "eval_steps_per_second": 0.323,
+      "epoch": 0.86,
+      "eval_logits/chosen": -4.706819534301758,
+      "eval_logits/rejected": -4.936363697052002,
+      "eval_logps/chosen": -405.887939453125,
+      "eval_logps/rejected": -521.2874755859375,
+      "eval_loss": 1.9827619791030884,
+      "eval_rewards/accuracies": 0.35546875,
+      "eval_rewards/chosen": -3.4222922325134277,
+      "eval_rewards/margins": -1.3097174167633057,
+      "eval_rewards/rejected": -2.112574815750122,
+      "eval_runtime": 97.4214,
+      "eval_samples_per_second": 20.529,
+      "eval_steps_per_second": 0.328,
       "step": 300
     },
     {
-      "epoch": 0.8168642951251647,
-      "grad_norm": 1030.5057638496492,
-      "learning_rate": 4.8834274139883084e-08,
-      "logits/chosen": -4.663365840911865,
-      "logits/rejected": -5.023074150085449,
-      "logps/chosen": -201.54336547851562,
-      "logps/rejected": -171.71324157714844,
-      "loss": 0.533,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 4.074584007263184,
-      "rewards/margins": 8.798391342163086,
-      "rewards/rejected": -4.723808765411377,
+      "epoch": 0.89,
+      "grad_norm": 1000.5341122694366,
+      "learning_rate": 1.8791541175240787e-09,
+      "logits/chosen": -4.3908562660217285,
+      "logits/rejected": -4.607417583465576,
+      "logps/chosen": -301.6288146972656,
+      "logps/rejected": -246.2473907470703,
+      "loss": 0.2942,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.3309147357940674,
+      "rewards/margins": 2.577249765396118,
+      "rewards/rejected": 0.753665030002594,
       "step": 310
     },
     {
-      "epoch": 0.8432147562582345,
-      "grad_norm": 1279.040936993484,
-      "learning_rate": 3.60317709937693e-08,
-      "logits/chosen": -4.707262992858887,
-      "logits/rejected": -5.011012077331543,
-      "logps/chosen": -223.0199432373047,
-      "logps/rejected": -183.51393127441406,
-      "loss": 0.5239,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 3.1946961879730225,
-      "rewards/margins": 7.443505764007568,
-      "rewards/rejected": -4.248808860778809,
+      "epoch": 0.92,
+      "grad_norm": 942.4369602130394,
+      "learning_rate": 1.0449549049596136e-09,
+      "logits/chosen": -4.230744361877441,
+      "logits/rejected": -4.579751968383789,
+      "logps/chosen": -287.4718322753906,
+      "logps/rejected": -232.3734893798828,
+      "loss": 0.3078,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.2846877574920654,
+      "rewards/margins": 2.4689810276031494,
+      "rewards/rejected": 0.8157066106796265,
       "step": 320
     },
     {
-      "epoch": 0.8695652173913043,
-      "grad_norm": 790.9511672313881,
-      "learning_rate": 2.5044085842905683e-08,
-      "logits/chosen": -4.707150459289551,
-      "logits/rejected": -4.967694282531738,
-      "logps/chosen": -208.9776153564453,
-      "logps/rejected": -184.1087188720703,
-      "loss": 0.5674,
-      "rewards/accuracies": 0.918749988079071,
-      "rewards/chosen": 4.723270893096924,
-      "rewards/margins": 9.567978858947754,
-      "rewards/rejected": -4.844708442687988,
+      "epoch": 0.94,
+      "grad_norm": 1041.373917696444,
+      "learning_rate": 4.5034924794443707e-10,
+      "logits/chosen": -4.3931074142456055,
+      "logits/rejected": -4.620477199554443,
+      "logps/chosen": -280.0606994628906,
+      "logps/rejected": -233.14346313476562,
+      "loss": 0.2917,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 3.2919604778289795,
+      "rewards/margins": 2.5820116996765137,
+      "rewards/rejected": 0.7099487781524658,
       "step": 330
     },
     {
-      "epoch": 0.8959156785243741,
-      "grad_norm": 1075.929120672392,
-      "learning_rate": 1.5964413124758493e-08,
-      "logits/chosen": -4.645999431610107,
-      "logits/rejected": -4.947402477264404,
-      "logps/chosen": -212.19015502929688,
-      "logps/rejected": -185.76678466796875,
-      "loss": 0.5578,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 4.216076374053955,
-      "rewards/margins": 8.17945671081543,
-      "rewards/rejected": -3.9633796215057373,
+      "epoch": 0.97,
+      "grad_norm": 859.9709811153568,
+      "learning_rate": 1.0128427297940723e-10,
+      "logits/chosen": -4.328027248382568,
+      "logits/rejected": -4.5136332511901855,
+      "logps/chosen": -284.2748718261719,
+      "logps/rejected": -244.03817749023438,
+      "loss": 0.2952,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 3.204352855682373,
+      "rewards/margins": 2.4567666053771973,
+      "rewards/rejected": 0.7475861310958862,
       "step": 340
     },
     {
-      "epoch": 0.922266139657444,
-      "grad_norm": 1055.0113319220882,
-      "learning_rate": 8.869764055041501e-09,
-      "logits/chosen": -4.714905738830566,
-      "logits/rejected": -4.937991619110107,
-      "logps/chosen": -215.5680389404297,
-      "logps/rejected": -200.6318359375,
-      "loss": 0.5113,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 2.9399425983428955,
-      "rewards/margins": 7.131015777587891,
-      "rewards/rejected": -4.191073417663574,
-      "step": 350
-    },
-    {
-      "epoch": 0.9486166007905138,
-      "grad_norm": 1003.3923870155849,
-      "learning_rate": 3.82031344036729e-09,
-      "logits/chosen": -4.614500999450684,
-      "logits/rejected": -4.9195661544799805,
-      "logps/chosen": -216.197998046875,
-      "logps/rejected": -186.8596649169922,
-      "loss": 0.4994,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 2.650672197341919,
-      "rewards/margins": 6.149393081665039,
-      "rewards/rejected": -3.49872088432312,
-      "step": 360
-    },
-    {
-      "epoch": 0.9749670619235836,
-      "grad_norm": 1021.842139855702,
-      "learning_rate": 8.588892925590063e-10,
-      "logits/chosen": -4.69917631149292,
-      "logits/rejected": -5.054296016693115,
-      "logps/chosen": -212.79714965820312,
-      "logps/rejected": -171.7442169189453,
-      "loss": 0.5535,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 3.2063796520233154,
-      "rewards/margins": 8.728368759155273,
-      "rewards/rejected": -5.521987438201904,
-      "step": 370
-    },
-    {
-      "epoch": 0.9986824769433466,
-      "step": 379,
+      "epoch": 1.0,
+      "step": 349,
       "total_flos": 0.0,
-      "train_loss": 0.5517442987587962,
-      "train_runtime": 6181.8185,
-      "train_samples_per_second": 7.85,
-      "train_steps_per_second": 0.061
+      "train_loss": 0.3696309270011661,
+      "train_runtime": 5591.2149,
+      "train_samples_per_second": 7.991,
+      "train_steps_per_second": 0.062
     }
   ],
   "logging_steps": 10,
-  "max_steps": 379,
+  "max_steps": 349,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
-  "stateful_callbacks": {
-    "TrainerControl": {
-      "args": {
-        "should_epoch_stop": false,
-        "should_evaluate": false,
-        "should_log": false,
-        "should_save": true,
-        "should_training_stop": false
-      },
-      "attributes": {}
-    }
-  },
   "total_flos": 0.0,
   "train_batch_size": 8,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df9dd9975041330cb3031003f27f8952f7657e67478afc48ee43616047d272ea
-size 6520
+oid sha256:dadd16e859cce8223cbc036d0b14a858573a3c16794cd0290547847afa9ddb52
+size 6264