Model save

README.md

@@ -16,14 +16,14 @@ should probably proofread and complete it, then remove this comment. -->
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
 - Loss: 0.6931
-- Rewards/chosen: -1.9201
-- Rewards/rejected: -1.9201
+- Rewards/chosen: -1.6342
+- Rewards/rejected: -1.6342
 - Rewards/accuracies: 0.0
 - Rewards/margins: 0.0
-- Logps/rejected: -159.4172
-- Logps/chosen: -159.4172
-- Logits/rejected: -3.2283
-- Logits/chosen: -3.2283
+- Logps/rejected: -158.8454
+- Logps/chosen: -158.8454
+- Logits/rejected: -3.2278
+- Logits/chosen: -3.2278
 
 ## Model description
 
@@ -53,21 +53,20 @@ The following hyperparameters were used during training:
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
-- lr_scheduler_warmup_ratio: 0.1
 - num_epochs: 1
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.3536        | 0.29  | 100  | 0.6931          | -1.3869        | -1.3869          | 0.0                | 0.0             | -158.3509      | -158.3509    | -3.2370         | -3.2370       |
-| 0.3091        | 0.57  | 200  | 0.6931          | -1.8814        | -1.8814          | 0.0                | 0.0             | -159.3398      | -159.3398    | -3.2304         | -3.2304       |
-| 0.3363        | 0.86  | 300  | 0.6931          | -1.9201        | -1.9201          | 0.0                | 0.0             | -159.4172      | -159.4172    | -3.2283         | -3.2283       |
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.3253        | 0.2857 | 100  | 0.6931          | -1.6970        | -1.6970          | 0.0                | 0.0             | -158.9711      | -158.9711    | -3.2366         | -3.2366       |
+| 0.3071        | 0.5714 | 200  | 0.6931          | -2.0914        | -2.0914          | 0.0                | 0.0             | -159.7597      | -159.7597    | -3.2287         | -3.2287       |
+| 0.3336        | 0.8571 | 300  | 0.6931          | -1.6342        | -1.6342          | 0.0                | 0.0             | -158.8454      | -158.8454    | -3.2278         | -3.2278       |
 
 
 ### Framework versions
 
-- Transformers 4.38.2
+- Transformers 4.41.1
 - Pytorch 2.1.2+cu118
 - Datasets 2.16.1
-- Tokenizers 0.15.2
+- Tokenizers 0.19.1

all_results.json

@@ -1,8 +1,9 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.37015721593584333,
-    "train_runtime": 5302.4484,
+    "total_flos": 0.0,
+    "train_loss": 0.34604193687438967,
+    "train_runtime": 5793.0735,
     "train_samples": 44755,
-    "train_samples_per_second": 8.44,
-    "train_steps_per_second": 0.066
+    "train_samples_per_second": 7.726,
+    "train_steps_per_second": 0.06
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.38.2",
+  "transformers_version": "4.41.1",
   "use_cache": false,
   "vocab_size": 32002
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
-  "transformers_version": "4.38.2"
+  "transformers_version": "4.41.1"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f8493cd3d8633107f37f766933768b972a71580f90b096571fcc9f7f95e4a96
+oid sha256:5620686540e3b69a8dff87d7dbb09524fd94d7f2126dc65a769fe4d670bd4a90
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f5824ec54fb5bda3f68132a93217d0b800db90a78fb0515b4b723ed929035a3
+oid sha256:0e57f83d8c347274f3c77cf169473468937aa6cee843f09e0709b1c3248e0fb8
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d89f85d62fe83d0e63f9373321633097e7aab9bbb8ae37954505ffd93eb3af47
+oid sha256:639c1f7e701b304b9eb6019aefbf2f2172c8a61237b358c0f6da984843c038be
 size 4540532728

runs/May31_01-13-18_n136-082-130/events.out.tfevents.1717089327.n136-082-130.3730659.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a615e1890f482937e0071599f0ffc9c51798e79a0ce8a2f8ca23e8b4e559918f
-size 28579
+oid sha256:6d7ceb3534c06a6da90d2635eb77b149335443213d3e7abafd6753ad70d80c9f
+size 32373

tokenizer.json

@@ -152,6 +152,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,8 +1,9 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.37015721593584333,
-    "train_runtime": 5302.4484,
+    "total_flos": 0.0,
+    "train_loss": 0.34604193687438967,
+    "train_runtime": 5793.0735,
     "train_samples": 44755,
-    "train_samples_per_second": 8.44,
-    "train_steps_per_second": 0.066
+    "train_samples_per_second": 7.726,
+    "train_steps_per_second": 0.06
 }

trainer_state.json

@@ -9,9 +9,9 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "grad_norm": 1533.9529579439338,
-      "learning_rate": 1.4285714285714284e-09,
+      "epoch": 0.002857142857142857,
+      "grad_norm": 1533.819121390321,
+      "learning_rate": 4.9998992904271775e-08,
       "logits/chosen": -4.185730934143066,
       "logits/rejected": -4.509836196899414,
       "logps/chosen": -274.000732421875,
@@ -24,586 +24,586 @@
       "step": 1
     },
     {
-      "epoch": 0.03,
-      "grad_norm": 1798.989663851012,
-      "learning_rate": 1.4285714285714284e-08,
-      "logits/chosen": -4.211680889129639,
-      "logits/rejected": -4.4850640296936035,
-      "logps/chosen": -318.5819091796875,
-      "logps/rejected": -257.2215270996094,
-      "loss": 0.7173,
-      "rewards/accuracies": 0.3888888955116272,
-      "rewards/chosen": -0.03539733216166496,
-      "rewards/margins": -0.0653969869017601,
-      "rewards/rejected": 0.02999965287744999,
+      "epoch": 0.02857142857142857,
+      "grad_norm": 1668.215158817105,
+      "learning_rate": 4.9899357349880975e-08,
+      "logits/chosen": -4.21472692489624,
+      "logits/rejected": -4.488520622253418,
+      "logps/chosen": -318.4333801269531,
+      "logps/rejected": -257.2440185546875,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.4930555522441864,
+      "rewards/chosen": 0.03888694569468498,
+      "rewards/margins": 0.020132217556238174,
+      "rewards/rejected": 0.018754728138446808,
       "step": 10
     },
     {
-      "epoch": 0.06,
-      "grad_norm": 1668.4002920396833,
-      "learning_rate": 2.857142857142857e-08,
-      "logits/chosen": -4.237612724304199,
-      "logits/rejected": -4.492175102233887,
-      "logps/chosen": -305.21356201171875,
-      "logps/rejected": -244.23391723632812,
-      "loss": 0.7172,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.011527794413268566,
-      "rewards/margins": 0.015434956178069115,
-      "rewards/rejected": -0.003907163627445698,
+      "epoch": 0.05714285714285714,
+      "grad_norm": 1345.3140295690048,
+      "learning_rate": 4.959823971496574e-08,
+      "logits/chosen": -4.247828006744385,
+      "logits/rejected": -4.502226829528809,
+      "logps/chosen": -304.4950866699219,
+      "logps/rejected": -244.0281219482422,
+      "loss": 0.6107,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.37075769901275635,
+      "rewards/margins": 0.27177533507347107,
+      "rewards/rejected": 0.09898237138986588,
       "step": 20
     },
     {
-      "epoch": 0.09,
-      "grad_norm": 1590.2675307754077,
-      "learning_rate": 4.285714285714285e-08,
-      "logits/chosen": -4.289905548095703,
-      "logits/rejected": -4.550080299377441,
-      "logps/chosen": -306.20733642578125,
-      "logps/rejected": -259.1973571777344,
-      "loss": 0.6742,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": 0.15369151532649994,
-      "rewards/margins": 0.08493399620056152,
-      "rewards/rejected": 0.06875751912593842,
+      "epoch": 0.08571428571428572,
+      "grad_norm": 1125.2250514399805,
+      "learning_rate": 4.9099071517396326e-08,
+      "logits/chosen": -4.306983947753906,
+      "logits/rejected": -4.5679826736450195,
+      "logps/chosen": -304.89337158203125,
+      "logps/rejected": -258.8996887207031,
+      "loss": 0.5148,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 0.8106807470321655,
+      "rewards/margins": 0.5930811762809753,
+      "rewards/rejected": 0.21759963035583496,
       "step": 30
     },
     {
-      "epoch": 0.11,
-      "grad_norm": 1234.2347053418307,
-      "learning_rate": 4.996892303047306e-08,
-      "logits/chosen": -4.294736862182617,
-      "logits/rejected": -4.560345649719238,
-      "logps/chosen": -294.7554931640625,
-      "logps/rejected": -234.93032836914062,
-      "loss": 0.5901,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.3894590735435486,
-      "rewards/margins": 0.2519429624080658,
-      "rewards/rejected": 0.1375161111354828,
+      "epoch": 0.11428571428571428,
+      "grad_norm": 1094.2742476604449,
+      "learning_rate": 4.8405871765993426e-08,
+      "logits/chosen": -4.310123443603516,
+      "logits/rejected": -4.577700614929199,
+      "logps/chosen": -292.9500427246094,
+      "logps/rejected": -234.384765625,
+      "loss": 0.4433,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 1.2921748161315918,
+      "rewards/margins": 0.8818836212158203,
+      "rewards/rejected": 0.41029104590415955,
       "step": 40
     },
     {
-      "epoch": 0.14,
-      "grad_norm": 1024.357051064495,
-      "learning_rate": 4.972077065562821e-08,
-      "logits/chosen": -4.300290584564209,
-      "logits/rejected": -4.545838832855225,
-      "logps/chosen": -300.6907043457031,
-      "logps/rejected": -252.9388427734375,
-      "loss": 0.4845,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 1.0174586772918701,
-      "rewards/margins": 0.6066709160804749,
-      "rewards/rejected": 0.41078776121139526,
+      "epoch": 0.14285714285714285,
+      "grad_norm": 866.0178113474443,
+      "learning_rate": 4.7524221697560474e-08,
+      "logits/chosen": -4.307633876800537,
+      "logits/rejected": -4.553541660308838,
+      "logps/chosen": -298.9430847167969,
+      "logps/rejected": -252.4582977294922,
+      "loss": 0.3906,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 1.8912757635116577,
+      "rewards/margins": 1.240208387374878,
+      "rewards/rejected": 0.6510674357414246,
       "step": 50
     },
     {
-      "epoch": 0.17,
-      "grad_norm": 993.3434930666515,
-      "learning_rate": 4.922693215572695e-08,
-      "logits/chosen": -4.309796333312988,
-      "logits/rejected": -4.538843154907227,
-      "logps/chosen": -308.38818359375,
-      "logps/rejected": -261.2379150390625,
-      "loss": 0.4421,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": 1.4816973209381104,
-      "rewards/margins": 0.7946016192436218,
-      "rewards/rejected": 0.6870955228805542,
+      "epoch": 0.17142857142857143,
+      "grad_norm": 837.0640990054532,
+      "learning_rate": 4.646121984004665e-08,
+      "logits/chosen": -4.309387683868408,
+      "logits/rejected": -4.5390214920043945,
+      "logps/chosen": -307.09820556640625,
+      "logps/rejected": -260.90191650390625,
+      "loss": 0.3814,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.126704216003418,
+      "rewards/margins": 1.2716383934020996,
+      "rewards/rejected": 0.8550659418106079,
       "step": 60
     },
     {
       "epoch": 0.2,
-      "grad_norm": 1067.0305947544678,
-      "learning_rate": 4.849231551964771e-08,
-      "logits/chosen": -4.422641277313232,
-      "logits/rejected": -4.553921699523926,
-      "logps/chosen": -285.1670227050781,
-      "logps/rejected": -236.08352661132812,
-      "loss": 0.4036,
+      "grad_norm": 985.5848045316892,
+      "learning_rate": 4.522542485937369e-08,
+      "logits/chosen": -4.4262213706970215,
+      "logits/rejected": -4.558186054229736,
+      "logps/chosen": -284.1617736816406,
+      "logps/rejected": -235.8411102294922,
+      "loss": 0.3695,
       "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 1.8685623407363892,
-      "rewards/margins": 1.2992055416107178,
-      "rewards/rejected": 0.5693566203117371,
+      "rewards/chosen": 2.3711953163146973,
+      "rewards/margins": 1.6806213855743408,
+      "rewards/rejected": 0.690574049949646,
       "step": 70
     },
     {
-      "epoch": 0.23,
-      "grad_norm": 1032.775358630634,
-      "learning_rate": 4.7524221697560474e-08,
-      "logits/chosen": -4.270499229431152,
-      "logits/rejected": -4.5226149559021,
-      "logps/chosen": -291.38165283203125,
-      "logps/rejected": -239.2751922607422,
-      "loss": 0.379,
+      "epoch": 0.22857142857142856,
+      "grad_norm": 951.7016793785169,
+      "learning_rate": 4.3826786650090273e-08,
+      "logits/chosen": -4.280414581298828,
+      "logits/rejected": -4.534539222717285,
+      "logps/chosen": -290.68597412109375,
+      "logps/rejected": -239.3174591064453,
+      "loss": 0.3466,
       "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 2.1530368328094482,
-      "rewards/margins": 1.5649009943008423,
-      "rewards/rejected": 0.5881360769271851,
+      "rewards/chosen": 2.500866413116455,
+      "rewards/margins": 1.933850646018982,
+      "rewards/rejected": 0.5670154094696045,
       "step": 80
     },
     {
-      "epoch": 0.26,
-      "grad_norm": 872.5516189664596,
-      "learning_rate": 4.633227204080389e-08,
-      "logits/chosen": -4.196683406829834,
-      "logits/rejected": -4.429889678955078,
-      "logps/chosen": -302.6337890625,
-      "logps/rejected": -257.6473388671875,
-      "loss": 0.3672,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 2.4647421836853027,
-      "rewards/margins": 1.6267496347427368,
-      "rewards/rejected": 0.8379926681518555,
+      "epoch": 0.2571428571428571,
+      "grad_norm": 868.4695107443264,
+      "learning_rate": 4.2276566224671614e-08,
+      "logits/chosen": -4.205895900726318,
+      "logits/rejected": -4.440802574157715,
+      "logps/chosen": -301.97674560546875,
+      "logps/rejected": -257.63153076171875,
+      "loss": 0.3457,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 2.7932560443878174,
+      "rewards/margins": 1.9473575353622437,
+      "rewards/rejected": 0.8458986282348633,
       "step": 90
     },
     {
-      "epoch": 0.29,
-      "grad_norm": 1367.4563539236096,
-      "learning_rate": 4.4928312680573064e-08,
-      "logits/chosen": -4.303212642669678,
-      "logits/rejected": -4.523682594299316,
-      "logps/chosen": -290.59271240234375,
-      "logps/rejected": -240.2918701171875,
-      "loss": 0.3536,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 2.6176071166992188,
-      "rewards/margins": 1.6629337072372437,
-      "rewards/rejected": 0.9546731114387512,
+      "epoch": 0.2857142857142857,
+      "grad_norm": 1324.6980610698608,
+      "learning_rate": 4.058724504646834e-08,
+      "logits/chosen": -4.310187816619873,
+      "logits/rejected": -4.531655311584473,
+      "logps/chosen": -290.22113037109375,
+      "logps/rejected": -240.4936981201172,
+      "loss": 0.3253,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 2.803382396697998,
+      "rewards/margins": 1.9496362209320068,
+      "rewards/rejected": 0.8537459373474121,
       "step": 100
     },
     {
-      "epoch": 0.29,
-      "eval_logits/chosen": -3.2369751930236816,
-      "eval_logits/rejected": -3.2369751930236816,
-      "eval_logps/chosen": -158.35089111328125,
-      "eval_logps/rejected": -158.35089111328125,
+      "epoch": 0.2857142857142857,
+      "eval_logits/chosen": -3.2365729808807373,
+      "eval_logits/rejected": -3.2365729808807373,
+      "eval_logps/chosen": -158.97109985351562,
+      "eval_logps/rejected": -158.97109985351562,
       "eval_loss": 0.6931471824645996,
       "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": -1.3869400024414062,
+      "eval_rewards/chosen": -1.697042465209961,
       "eval_rewards/margins": 0.0,
-      "eval_rewards/rejected": -1.3869400024414062,
-      "eval_runtime": 1.4807,
-      "eval_samples_per_second": 0.675,
-      "eval_steps_per_second": 0.675,
+      "eval_rewards/rejected": -1.697042465209961,
+      "eval_runtime": 1.5274,
+      "eval_samples_per_second": 0.655,
+      "eval_steps_per_second": 0.655,
       "step": 100
     },
     {
-      "epoch": 0.31,
-      "grad_norm": 934.0407259676991,
-      "learning_rate": 4.3326296795745654e-08,
-      "logits/chosen": -4.324947357177734,
-      "logits/rejected": -4.566166400909424,
-      "logps/chosen": -297.800537109375,
-      "logps/rejected": -249.71450805664062,
-      "loss": 0.3471,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 2.8152554035186768,
-      "rewards/margins": 2.0201711654663086,
-      "rewards/rejected": 0.7950841784477234,
+      "epoch": 0.3142857142857143,
+      "grad_norm": 904.1535779728154,
+      "learning_rate": 3.8772424536302564e-08,
+      "logits/chosen": -4.3289313316345215,
+      "logits/rejected": -4.571288108825684,
+      "logps/chosen": -297.5599060058594,
+      "logps/rejected": -249.8257293701172,
+      "loss": 0.345,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.935572385787964,
+      "rewards/margins": 2.1961002349853516,
+      "rewards/rejected": 0.7394723892211914,
       "step": 110
     },
     {
-      "epoch": 0.34,
-      "grad_norm": 1096.418461846408,
-      "learning_rate": 4.1542145939921484e-08,
-      "logits/chosen": -4.368184566497803,
-      "logits/rejected": -4.613701820373535,
-      "logps/chosen": -287.2909851074219,
-      "logps/rejected": -237.5634002685547,
-      "loss": 0.3344,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 2.821599245071411,
-      "rewards/margins": 2.0445477962493896,
-      "rewards/rejected": 0.7770514488220215,
+      "epoch": 0.34285714285714286,
+      "grad_norm": 1086.0405455439977,
+      "learning_rate": 3.6846716561824964e-08,
+      "logits/chosen": -4.373476982116699,
+      "logits/rejected": -4.619527816772461,
+      "logps/chosen": -287.13385009765625,
+      "logps/rejected": -237.6899871826172,
+      "loss": 0.3263,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 2.900177478790283,
+      "rewards/margins": 2.18641996383667,
+      "rewards/rejected": 0.7137576937675476,
       "step": 120
     },
     {
-      "epoch": 0.37,
-      "grad_norm": 722.1338343165834,
-      "learning_rate": 3.959359180586975e-08,
-      "logits/chosen": -4.369457244873047,
-      "logits/rejected": -4.6166582107543945,
-      "logps/chosen": -277.1622314453125,
-      "logps/rejected": -218.28323364257812,
-      "loss": 0.3351,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 2.7689156532287598,
-      "rewards/margins": 1.9403083324432373,
-      "rewards/rejected": 0.8286076784133911,
+      "epoch": 0.37142857142857144,
+      "grad_norm": 753.3076911412429,
+      "learning_rate": 3.482562579134809e-08,
+      "logits/chosen": -4.375483512878418,
+      "logits/rejected": -4.624612331390381,
+      "logps/chosen": -276.9853515625,
+      "logps/rejected": -218.39633178710938,
+      "loss": 0.3228,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 2.8573689460754395,
+      "rewards/margins": 2.0853238105773926,
+      "rewards/rejected": 0.7720457315444946,
       "step": 130
     },
     {
       "epoch": 0.4,
-      "grad_norm": 854.554160074848,
-      "learning_rate": 3.75e-08,
-      "logits/chosen": -4.2873382568359375,
-      "logits/rejected": -4.604461669921875,
-      "logps/chosen": -294.3111267089844,
-      "logps/rejected": -239.65628051757812,
-      "loss": 0.3544,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 2.786259174346924,
-      "rewards/margins": 2.21262526512146,
-      "rewards/rejected": 0.5736337900161743,
+      "grad_norm": 840.6908677183937,
+      "learning_rate": 3.272542485937369e-08,
+      "logits/chosen": -4.290375232696533,
+      "logits/rejected": -4.607089996337891,
+      "logps/chosen": -294.1933288574219,
+      "logps/rejected": -239.8271942138672,
+      "loss": 0.3428,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.845167875289917,
+      "rewards/margins": 2.3569939136505127,
+      "rewards/rejected": 0.48817411065101624,
       "step": 140
     },
     {
-      "epoch": 0.43,
-      "grad_norm": 1024.227354138342,
-      "learning_rate": 3.5282177578265295e-08,
-      "logits/chosen": -4.251392364501953,
-      "logits/rejected": -4.515843391418457,
-      "logps/chosen": -293.588623046875,
-      "logps/rejected": -250.3025360107422,
-      "loss": 0.3157,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 2.910778045654297,
-      "rewards/margins": 2.2114222049713135,
-      "rewards/rejected": 0.6993557810783386,
+      "epoch": 0.42857142857142855,
+      "grad_norm": 1010.0880176318159,
+      "learning_rate": 3.056302334890786e-08,
+      "logits/chosen": -4.25304651260376,
+      "logits/rejected": -4.517129421234131,
+      "logps/chosen": -293.35797119140625,
+      "logps/rejected": -250.48641967773438,
+      "loss": 0.313,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 3.026089906692505,
+      "rewards/margins": 2.4186887741088867,
+      "rewards/rejected": 0.6074013113975525,
       "step": 150
     },
     {
-      "epoch": 0.46,
-      "grad_norm": 958.7773617526785,
-      "learning_rate": 3.296216625629211e-08,
-      "logits/chosen": -4.2807087898254395,
-      "logits/rejected": -4.571825981140137,
-      "logps/chosen": -294.7529296875,
-      "logps/rejected": -235.1978759765625,
-      "loss": 0.3382,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 3.0874907970428467,
-      "rewards/margins": 2.296449899673462,
-      "rewards/rejected": 0.7910411953926086,
+      "epoch": 0.45714285714285713,
+      "grad_norm": 881.3705397776472,
+      "learning_rate": 2.8355831645441387e-08,
+      "logits/chosen": -4.279029369354248,
+      "logits/rejected": -4.569698810577393,
+      "logps/chosen": -294.6650695800781,
+      "logps/rejected": -235.29067993164062,
+      "loss": 0.338,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 3.13142728805542,
+      "rewards/margins": 2.386794090270996,
+      "rewards/rejected": 0.7446335554122925,
       "step": 160
     },
     {
-      "epoch": 0.49,
-      "grad_norm": 844.9738579712869,
-      "learning_rate": 3.056302334890786e-08,
-      "logits/chosen": -4.230380058288574,
-      "logits/rejected": -4.466236114501953,
-      "logps/chosen": -294.3287658691406,
-      "logps/rejected": -241.07357788085938,
-      "loss": 0.3284,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 3.165980815887451,
-      "rewards/margins": 2.4775989055633545,
-      "rewards/rejected": 0.6883817911148071,
+      "epoch": 0.4857142857142857,
+      "grad_norm": 857.1741181734847,
+      "learning_rate": 2.6121620758762875e-08,
+      "logits/chosen": -4.228262901306152,
+      "logits/rejected": -4.463376522064209,
+      "logps/chosen": -294.2881774902344,
+      "logps/rejected": -241.1619415283203,
+      "loss": 0.3264,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.1862380504608154,
+      "rewards/margins": 2.542022228240967,
+      "rewards/rejected": 0.6442161202430725,
       "step": 170
     },
     {
-      "epoch": 0.51,
-      "grad_norm": 878.602697816289,
-      "learning_rate": 2.8108592616187133e-08,
-      "logits/chosen": -4.364335060119629,
-      "logits/rejected": -4.596997261047363,
-      "logps/chosen": -283.4359130859375,
-      "logps/rejected": -240.91796875,
-      "loss": 0.3163,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 3.220620632171631,
-      "rewards/margins": 2.4390523433685303,
-      "rewards/rejected": 0.7815683484077454,
+      "epoch": 0.5142857142857142,
+      "grad_norm": 923.726950031985,
+      "learning_rate": 2.3878379241237133e-08,
+      "logits/chosen": -4.36321496963501,
+      "logits/rejected": -4.595080375671387,
+      "logps/chosen": -283.4067687988281,
+      "logps/rejected": -241.1226043701172,
+      "loss": 0.3098,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 3.2351880073547363,
+      "rewards/margins": 2.5559356212615967,
+      "rewards/rejected": 0.679252028465271,
       "step": 180
     },
     {
-      "epoch": 0.54,
-      "grad_norm": 831.8981359537083,
-      "learning_rate": 2.562326729345182e-08,
-      "logits/chosen": -4.325879096984863,
-      "logits/rejected": -4.494770526885986,
-      "logps/chosen": -306.01171875,
-      "logps/rejected": -257.24298095703125,
-      "loss": 0.2864,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 3.1413779258728027,
-      "rewards/margins": 2.4198684692382812,
-      "rewards/rejected": 0.7215089797973633,
+      "epoch": 0.5428571428571428,
+      "grad_norm": 792.8076625116224,
+      "learning_rate": 2.164416835455862e-08,
+      "logits/chosen": -4.323281288146973,
+      "logits/rejected": -4.492599964141846,
+      "logps/chosen": -305.88592529296875,
+      "logps/rejected": -257.3919982910156,
+      "loss": 0.2759,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 3.2042744159698486,
+      "rewards/margins": 2.557298421859741,
+      "rewards/rejected": 0.6469758152961731,
       "step": 190
     },
     {
-      "epoch": 0.57,
-      "grad_norm": 752.1347907356251,
-      "learning_rate": 2.3131747660339392e-08,
-      "logits/chosen": -4.358603000640869,
-      "logits/rejected": -4.599839687347412,
-      "logps/chosen": -290.7633972167969,
-      "logps/rejected": -249.0517120361328,
-      "loss": 0.3091,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 757.2657183486399,
+      "learning_rate": 1.943697665109214e-08,
+      "logits/chosen": -4.354622840881348,
+      "logits/rejected": -4.594660758972168,
+      "logps/chosen": -290.7183532714844,
+      "logps/rejected": -249.0703582763672,
+      "loss": 0.3071,
       "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 3.1841001510620117,
-      "rewards/margins": 2.3024184703826904,
-      "rewards/rejected": 0.8816817998886108,
+      "rewards/chosen": 3.2066097259521484,
+      "rewards/margins": 2.334256649017334,
+      "rewards/rejected": 0.8723530769348145,
       "step": 200
     },
     {
-      "epoch": 0.57,
-      "eval_logits/chosen": -3.230353832244873,
-      "eval_logits/rejected": -3.230353832244873,
-      "eval_logps/chosen": -159.3397674560547,
-      "eval_logps/rejected": -159.3397674560547,
+      "epoch": 0.5714285714285714,
+      "eval_logits/chosen": -3.2287425994873047,
+      "eval_logits/rejected": -3.2287425994873047,
+      "eval_logps/chosen": -159.75973510742188,
+      "eval_logps/rejected": -159.75973510742188,
       "eval_loss": 0.6931471824645996,
       "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": -1.8813800811767578,
+      "eval_rewards/chosen": -2.0913619995117188,
       "eval_rewards/margins": 0.0,
-      "eval_rewards/rejected": -1.8813800811767578,
-      "eval_runtime": 1.4678,
-      "eval_samples_per_second": 0.681,
-      "eval_steps_per_second": 0.681,
+      "eval_rewards/rejected": -2.0913619995117188,
+      "eval_runtime": 1.503,
+      "eval_samples_per_second": 0.665,
+      "eval_steps_per_second": 0.665,
       "step": 200
     },
     {
       "epoch": 0.6,
-      "grad_norm": 873.0977005391466,
-      "learning_rate": 2.065879555832674e-08,
-      "logits/chosen": -4.293581962585449,
-      "logits/rejected": -4.587343692779541,
-      "logps/chosen": -304.852294921875,
-      "logps/rejected": -254.5345916748047,
-      "loss": 0.3125,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 3.1841378211975098,
-      "rewards/margins": 2.6728169918060303,
-      "rewards/rejected": 0.5113206505775452,
+      "grad_norm": 891.2229023091579,
+      "learning_rate": 1.7274575140626317e-08,
+      "logits/chosen": -4.2898783683776855,
+      "logits/rejected": -4.5833001136779785,
+      "logps/chosen": -304.8259582519531,
+      "logps/rejected": -254.60757446289062,
+      "loss": 0.3024,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 3.1973044872283936,
+      "rewards/margins": 2.722479820251465,
+      "rewards/rejected": 0.47482460737228394,
       "step": 210
     },
     {
-      "epoch": 0.63,
-      "grad_norm": 896.6576085052702,
-      "learning_rate": 1.8228988296424874e-08,
-      "logits/chosen": -4.245147705078125,
-      "logits/rejected": -4.591282367706299,
-      "logps/chosen": -295.0483703613281,
-      "logps/rejected": -224.8583984375,
-      "loss": 0.3117,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 3.232748031616211,
-      "rewards/margins": 2.8655002117156982,
-      "rewards/rejected": 0.367247611284256,
+      "epoch": 0.6285714285714286,
+      "grad_norm": 938.9689090425716,
+      "learning_rate": 1.517437420865191e-08,
+      "logits/chosen": -4.243287086486816,
+      "logits/rejected": -4.58882999420166,
+      "logps/chosen": -294.97943115234375,
+      "logps/rejected": -225.01730346679688,
+      "loss": 0.2916,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 3.2672336101531982,
+      "rewards/margins": 2.979445695877075,
+      "rewards/rejected": 0.2877880036830902,
       "step": 220
     },
     {
-      "epoch": 0.66,
-      "grad_norm": 896.8048647139675,
-      "learning_rate": 1.5866474390840123e-08,
-      "logits/chosen": -4.371578216552734,
-      "logits/rejected": -4.561336994171143,
-      "logps/chosen": -279.5164489746094,
-      "logps/rejected": -225.96630859375,
-      "loss": 0.2801,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 2.720884323120117,
-      "rewards/margins": 2.2243082523345947,
-      "rewards/rejected": 0.4965757727622986,
+      "epoch": 0.6571428571428571,
+      "grad_norm": 889.4233945739674,
+      "learning_rate": 1.3153283438175034e-08,
+      "logits/chosen": -4.367541313171387,
+      "logits/rejected": -4.557114601135254,
+      "logps/chosen": -279.52362060546875,
+      "logps/rejected": -226.0491180419922,
+      "loss": 0.2804,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 2.717285633087158,
+      "rewards/margins": 2.2621235847473145,
+      "rewards/rejected": 0.45516151189804077,
       "step": 230
     },
     {
-      "epoch": 0.69,
-      "grad_norm": 794.1572537064102,
-      "learning_rate": 1.3594733566170923e-08,
-      "logits/chosen": -4.393925666809082,
-      "logits/rejected": -4.711283206939697,
-      "logps/chosen": -256.6236877441406,
-      "logps/rejected": -215.0515594482422,
-      "loss": 0.3001,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 2.945354700088501,
-      "rewards/margins": 2.4272878170013428,
-      "rewards/rejected": 0.5180668830871582,
+      "epoch": 0.6857142857142857,
+      "grad_norm": 764.4075642697645,
+      "learning_rate": 1.1227575463697438e-08,
+      "logits/chosen": -4.391533851623535,
+      "logits/rejected": -4.709025859832764,
+      "logps/chosen": -256.52532958984375,
+      "logps/rejected": -215.2234649658203,
+      "loss": 0.287,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 2.9945430755615234,
+      "rewards/margins": 2.5624358654022217,
+      "rewards/rejected": 0.4321066439151764,
       "step": 240
     },
     {
-      "epoch": 0.71,
-      "grad_norm": 1028.7241893624246,
-      "learning_rate": 1.1436343403356017e-08,
-      "logits/chosen": -4.338345527648926,
-      "logits/rejected": -4.6125383377075195,
-      "logps/chosen": -276.8526611328125,
-      "logps/rejected": -232.35494995117188,
-      "loss": 0.296,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 2.86142897605896,
-      "rewards/margins": 2.335439443588257,
-      "rewards/rejected": 0.5259896516799927,
+      "epoch": 0.7142857142857143,
+      "grad_norm": 1078.8910476393369,
+      "learning_rate": 9.412754953531663e-09,
+      "logits/chosen": -4.338193416595459,
+      "logits/rejected": -4.612923622131348,
+      "logps/chosen": -276.8057861328125,
+      "logps/rejected": -232.43807983398438,
+      "loss": 0.2923,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.8848717212677,
+      "rewards/margins": 2.400455951690674,
+      "rewards/rejected": 0.4844156801700592,
       "step": 250
     },
     {
-      "epoch": 0.74,
-      "grad_norm": 955.6521893892625,
-      "learning_rate": 9.412754953531663e-09,
-      "logits/chosen": -4.384321212768555,
-      "logits/rejected": -4.630730628967285,
-      "logps/chosen": -269.96063232421875,
-      "logps/rejected": -240.08322143554688,
-      "loss": 0.2927,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 2.748866558074951,
-      "rewards/margins": 2.221247434616089,
-      "rewards/rejected": 0.5276187658309937,
+      "epoch": 0.7428571428571429,
+      "grad_norm": 888.6230207618648,
+      "learning_rate": 7.723433775328384e-09,
+      "logits/chosen": -4.384097099304199,
+      "logits/rejected": -4.630164623260498,
+      "logps/chosen": -269.947265625,
+      "logps/rejected": -240.2582244873047,
+      "loss": 0.2917,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.7555510997772217,
+      "rewards/margins": 2.3154397010803223,
+      "rewards/rejected": 0.4401116967201233,
       "step": 260
     },
     {
-      "epoch": 0.77,
-      "grad_norm": 861.1823956058561,
-      "learning_rate": 7.544079547848182e-09,
-      "logits/chosen": -4.517904281616211,
-      "logits/rejected": -4.6855998039245605,
-      "logps/chosen": -271.70074462890625,
-      "logps/rejected": -227.885009765625,
-      "loss": 0.3225,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 2.6885054111480713,
-      "rewards/margins": 2.017667293548584,
-      "rewards/rejected": 0.670837938785553,
+      "epoch": 0.7714285714285715,
+      "grad_norm": 839.6114448931011,
+      "learning_rate": 6.173213349909728e-09,
+      "logits/chosen": -4.516074180603027,
+      "logits/rejected": -4.6834282875061035,
+      "logps/chosen": -271.71820068359375,
+      "logps/rejected": -227.9601287841797,
+      "loss": 0.3199,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 2.6797890663146973,
+      "rewards/margins": 2.0464982986450195,
+      "rewards/rejected": 0.6332908868789673,
       "step": 270
     },
     {
       "epoch": 0.8,
-      "grad_norm": 732.1581791035536,
-      "learning_rate": 5.8488889220255525e-09,
-      "logits/chosen": -4.298913955688477,
-      "logits/rejected": -4.587487697601318,
-      "logps/chosen": -272.83935546875,
-      "logps/rejected": -229.5067596435547,
-      "loss": 0.2873,
-      "rewards/accuracies": 0.90625,
-      "rewards/chosen": 3.1228713989257812,
-      "rewards/margins": 2.495274066925049,
-      "rewards/rejected": 0.6275972723960876,
+      "grad_norm": 794.9828383467956,
+      "learning_rate": 4.7745751406263165e-09,
+      "logits/chosen": -4.297934532165527,
+      "logits/rejected": -4.586764335632324,
+      "logps/chosen": -272.8429870605469,
+      "logps/rejected": -229.56021118164062,
+      "loss": 0.2842,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 3.1210646629333496,
+      "rewards/margins": 2.520183801651001,
+      "rewards/rejected": 0.6008811593055725,
       "step": 280
     },
     {
-      "epoch": 0.83,
-      "grad_norm": 1125.5972329270946,
-      "learning_rate": 4.344030642100133e-09,
-      "logits/chosen": -4.320634365081787,
-      "logits/rejected": -4.5083842277526855,
-      "logps/chosen": -280.2475280761719,
-      "logps/rejected": -236.018798828125,
-      "loss": 0.3136,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.1591415405273438,
-      "rewards/margins": 2.4951727390289307,
-      "rewards/rejected": 0.6639689207077026,
+      "epoch": 0.8285714285714286,
+      "grad_norm": 1122.8170349728516,
+      "learning_rate": 3.5387801599533474e-09,
+      "logits/chosen": -4.3172149658203125,
+      "logits/rejected": -4.5055928230285645,
+      "logps/chosen": -280.2183837890625,
+      "logps/rejected": -236.1828155517578,
+      "loss": 0.3083,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 3.173701286315918,
+      "rewards/margins": 2.5917415618896484,
+      "rewards/rejected": 0.5819598436355591,
       "step": 290
     },
     {
-      "epoch": 0.86,
-      "grad_norm": 788.6539907080908,
-      "learning_rate": 3.0444606657442835e-09,
-      "logits/chosen": -4.298077583312988,
-      "logits/rejected": -4.541121959686279,
-      "logps/chosen": -296.12176513671875,
-      "logps/rejected": -239.7339324951172,
-      "loss": 0.3363,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 3.3548855781555176,
-      "rewards/margins": 2.5390803813934326,
-      "rewards/rejected": 0.8158050775527954,
+      "epoch": 0.8571428571428571,
+      "grad_norm": 746.1081306618772,
+      "learning_rate": 2.475778302439524e-09,
+      "logits/chosen": -4.29483699798584,
+      "logits/rejected": -4.537248134613037,
+      "logps/chosen": -296.06756591796875,
+      "logps/rejected": -239.85092163085938,
+      "loss": 0.3336,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.3819689750671387,
+      "rewards/margins": 2.6246695518493652,
+      "rewards/rejected": 0.7572996020317078,
       "step": 300
     },
     {
-      "epoch": 0.86,
-      "eval_logits/chosen": -3.2283477783203125,
-      "eval_logits/rejected": -3.2283477783203125,
-      "eval_logps/chosen": -159.417236328125,
-      "eval_logps/rejected": -159.417236328125,
+      "epoch": 0.8571428571428571,
+      "eval_logits/chosen": -3.2277979850769043,
+      "eval_logits/rejected": -3.2277979850769043,
+      "eval_logps/chosen": -158.84542846679688,
+      "eval_logps/rejected": -158.84542846679688,
       "eval_loss": 0.6931471824645996,
       "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": -1.9201126098632812,
+      "eval_rewards/chosen": -1.6342105865478516,
       "eval_rewards/margins": 0.0,
-      "eval_rewards/rejected": -1.9201126098632812,
-      "eval_runtime": 1.4723,
-      "eval_samples_per_second": 0.679,
-      "eval_steps_per_second": 0.679,
+      "eval_rewards/rejected": -1.6342105865478516,
+      "eval_runtime": 1.5181,
+      "eval_samples_per_second": 0.659,
+      "eval_steps_per_second": 0.659,
       "step": 300
     },
     {
-      "epoch": 0.89,
-      "grad_norm": 1232.483796740804,
-      "learning_rate": 1.9630947032398067e-09,
-      "logits/chosen": -4.439385414123535,
-      "logits/rejected": -4.579739093780518,
-      "logps/chosen": -260.4383850097656,
-      "logps/rejected": -226.1852264404297,
-      "loss": 0.2944,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 2.838869571685791,
-      "rewards/margins": 2.284980297088623,
-      "rewards/rejected": 0.5538893342018127,
+      "epoch": 0.8857142857142857,
+      "grad_norm": 1352.5130710151727,
+      "learning_rate": 1.5941282340065698e-09,
+      "logits/chosen": -4.436648368835449,
+      "logits/rejected": -4.576234817504883,
+      "logps/chosen": -260.4552001953125,
+      "logps/rejected": -226.2211456298828,
+      "loss": 0.2894,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 2.8304812908172607,
+      "rewards/margins": 2.2945332527160645,
+      "rewards/rejected": 0.535947859287262,
       "step": 310
     },
     {
-      "epoch": 0.91,
-      "grad_norm": 734.9918892690486,
-      "learning_rate": 1.1106798553464803e-09,
-      "logits/chosen": -4.274772644042969,
-      "logits/rejected": -4.528196334838867,
-      "logps/chosen": -290.4882507324219,
-      "logps/rejected": -243.47128295898438,
-      "loss": 0.3081,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 3.197766065597534,
-      "rewards/margins": 2.4614479541778564,
-      "rewards/rejected": 0.7363181114196777,
+      "epoch": 0.9142857142857143,
+      "grad_norm": 759.0330474241811,
+      "learning_rate": 9.009284826036689e-10,
+      "logits/chosen": -4.273808002471924,
+      "logits/rejected": -4.527020454406738,
+      "logps/chosen": -290.4542541503906,
+      "logps/rejected": -243.5283966064453,
+      "loss": 0.314,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 3.2147815227508545,
+      "rewards/margins": 2.507025718688965,
+      "rewards/rejected": 0.7077558040618896,
       "step": 320
     },
     {
-      "epoch": 0.94,
-      "grad_norm": 1013.5360319244875,
-      "learning_rate": 4.956878037864043e-10,
-      "logits/chosen": -4.329029083251953,
-      "logits/rejected": -4.53527307510376,
-      "logps/chosen": -303.6526794433594,
-      "logps/rejected": -251.791259765625,
-      "loss": 0.3026,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 2.9665093421936035,
-      "rewards/margins": 2.3622777462005615,
-      "rewards/rejected": 0.6042317152023315,
+      "epoch": 0.9428571428571428,
+      "grad_norm": 1012.631208038228,
+      "learning_rate": 4.017602850342583e-10,
+      "logits/chosen": -4.329623222351074,
+      "logits/rejected": -4.535677433013916,
+      "logps/chosen": -303.55926513671875,
+      "logps/rejected": -251.89437866210938,
+      "loss": 0.3017,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 3.013214588165283,
+      "rewards/margins": 2.4605515003204346,
+      "rewards/rejected": 0.5526631474494934,
       "step": 330
     },
     {
-      "epoch": 0.97,
-      "grad_norm": 1029.0631247014248,
-      "learning_rate": 1.2423061586496474e-10,
-      "logits/chosen": -4.317473888397217,
-      "logits/rejected": -4.609925270080566,
-      "logps/chosen": -283.091552734375,
-      "logps/rejected": -227.1355438232422,
-      "loss": 0.3064,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 2.9268617630004883,
-      "rewards/margins": 2.242302417755127,
-      "rewards/rejected": 0.6845596432685852,
+      "epoch": 0.9714285714285714,
+      "grad_norm": 1047.7085022489064,
+      "learning_rate": 1.0064265011902328e-10,
+      "logits/chosen": -4.3172688484191895,
+      "logits/rejected": -4.609848976135254,
+      "logps/chosen": -283.23883056640625,
+      "logps/rejected": -227.2130126953125,
+      "loss": 0.3029,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 2.853224277496338,
+      "rewards/margins": 2.2073826789855957,
+      "rewards/rejected": 0.6458419561386108,
       "step": 340
     },
     {
       "epoch": 1.0,
-      "grad_norm": 657.4804286362323,
+      "grad_norm": 570.6961447414922,
       "learning_rate": 0.0,
-      "logits/chosen": -4.288041114807129,
-      "logits/rejected": -4.490693092346191,
-      "logps/chosen": -289.8371887207031,
-      "logps/rejected": -243.97164916992188,
-      "loss": 0.2999,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 3.091984987258911,
-      "rewards/margins": 2.4618594646453857,
-      "rewards/rejected": 0.6301255226135254,
+      "logits/chosen": -4.2885661125183105,
+      "logits/rejected": -4.4927144050598145,
+      "logps/chosen": -289.79022216796875,
+      "logps/rejected": -244.1138916015625,
+      "loss": 0.2904,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 3.115464448928833,
+      "rewards/margins": 2.5564632415771484,
+      "rewards/rejected": 0.5590011477470398,
       "step": 350
     },
     {
       "epoch": 1.0,
       "step": 350,
       "total_flos": 0.0,
-      "train_loss": 0.37015721593584333,
-      "train_runtime": 5302.4484,
-      "train_samples_per_second": 8.44,
-      "train_steps_per_second": 0.066
+      "train_loss": 0.34604193687438967,
+      "train_runtime": 5793.0735,
+      "train_samples_per_second": 7.726,
+      "train_steps_per_second": 0.06
     }
   ],
   "logging_steps": 10,
@@ -611,6 +611,18 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
   "total_flos": 0.0,
   "train_batch_size": 8,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25686a2a94e778bd06e64992e121362fb8e396ce160a2ce4cb0004cc6efb88ff
-size 6264
+oid sha256:4cfcdc187e96bc5f72f21240b097691d6a8cdf8d0fcc5ad2fe20f43a43287471
+size 6520