Model save

README.md

@@ -16,14 +16,14 @@ should probably proofread and complete it, then remove this comment. -->
 This model was trained from scratch on the None dataset.
 It achieves the following results on the evaluation set:
 - Loss: 0.6931
-- Rewards/chosen: -1.6342
-- Rewards/rejected: -1.6342
+- Rewards/chosen: -8.4881
+- Rewards/rejected: -8.4881
 - Rewards/accuracies: 0.0
 - Rewards/margins: 0.0
-- Logps/rejected: -158.8454
-- Logps/chosen: -158.8454
-- Logits/rejected: -3.2278
-- Logits/chosen: -3.2278
+- Logps/rejected: -164.0651
+- Logps/chosen: -164.0651
+- Logits/rejected: -3.2224
+- Logits/chosen: -3.2224
 
 ## Model description
 
@@ -42,7 +42,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-08
+- learning_rate: 5e-07
 - train_batch_size: 8
 - eval_batch_size: 8
 - seed: 42
@@ -53,20 +53,21 @@ The following hyperparameters were used during training:
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
 - num_epochs: 1
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
-|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.3253        | 0.2857 | 100  | 0.6931          | -1.6970        | -1.6970          | 0.0                | 0.0             | -158.9711      | -158.9711    | -3.2366         | -3.2366       |
-| 0.3071        | 0.5714 | 200  | 0.6931          | -2.0914        | -2.0914          | 0.0                | 0.0             | -159.7597      | -159.7597    | -3.2287         | -3.2287       |
-| 0.3336        | 0.8571 | 300  | 0.6931          | -1.6342        | -1.6342          | 0.0                | 0.0             | -158.8454      | -158.8454    | -3.2278         | -3.2278       |
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.7723        | 0.29  | 100  | 0.6931          | -2.0247        | -2.0247          | 0.0                | 0.0             | -157.6017      | -157.6017    | -3.1472         | -3.1472       |
+| 0.7717        | 0.57  | 200  | 0.6931          | -12.0830       | -12.0830         | 0.0                | 0.0             | -167.6601      | -167.6601    | -3.1635         | -3.1635       |
+| 0.782         | 0.86  | 300  | 0.6931          | -8.4881        | -8.4881          | 0.0                | 0.0             | -164.0651      | -164.0651    | -3.2224         | -3.2224       |
 
 
 ### Framework versions
 
-- Transformers 4.41.1
+- Transformers 4.38.2
 - Pytorch 2.1.2+cu118
 - Datasets 2.16.1
-- Tokenizers 0.19.1
+- Tokenizers 0.15.2

all_results.json

@@ -1,9 +1,8 @@
 {
     "epoch": 1.0,
-    "total_flos": 0.0,
-    "train_loss": 0.34604193687438967,
-    "train_runtime": 5793.0735,
+    "train_loss": 0.7428068714482444,
+    "train_runtime": 5299.044,
     "train_samples": 44755,
-    "train_samples_per_second": 7.726,
-    "train_steps_per_second": 0.06
+    "train_samples_per_second": 8.446,
+    "train_steps_per_second": 0.066
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.41.1",
+  "transformers_version": "4.38.2",
   "use_cache": false,
   "vocab_size": 32002
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
-  "transformers_version": "4.41.1"
+  "transformers_version": "4.38.2"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5620686540e3b69a8dff87d7dbb09524fd94d7f2126dc65a769fe4d670bd4a90
+oid sha256:d8a16dcdeac389a7edb934e74708a95a01cd1e632e8f90a73780fef9e4ba9a79
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e57f83d8c347274f3c77cf169473468937aa6cee843f09e0709b1c3248e0fb8
+oid sha256:830c15b3f985c15ad7fd7fed750f2e2465dfc40ee9aa0da31053db52e903da60
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:639c1f7e701b304b9eb6019aefbf2f2172c8a61237b358c0f6da984843c038be
+oid sha256:a9abe98aec4f8efa4737df5455a2ed808d0f8ebad162d17a1674f84f71648f8d
 size 4540532728

runs/May31_01-40-50_n136-100-194/events.out.tfevents.1717090981.n136-100-194.1984510.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:99bc78ded1b39fbe01859f8d995f1b0573567e7d45edc6350ee4a1fa2d31ce95
-size 28312
+oid sha256:ec8a38bec5df134d51ff42aa4b854f577d10febb82aa267f83550844de67a812
+size 32106

tokenizer.json

@@ -152,7 +152,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,9 +1,8 @@
 {
     "epoch": 1.0,
-    "total_flos": 0.0,
-    "train_loss": 0.34604193687438967,
-    "train_runtime": 5793.0735,
+    "train_loss": 0.7428068714482444,
+    "train_runtime": 5299.044,
     "train_samples": 44755,
-    "train_samples_per_second": 7.726,
-    "train_steps_per_second": 0.06
+    "train_samples_per_second": 8.446,
+    "train_steps_per_second": 0.066
 }

trainer_state.json

@@ -9,14 +9,14 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.002857142857142857,
-      "grad_norm": 1533.819121390321,
-      "learning_rate": 4.9998992904271775e-08,
+      "epoch": 0.0,
+      "grad_norm": 3039.0846042522635,
+      "learning_rate": 1.4285714285714284e-08,
       "logits/chosen": -4.185730934143066,
       "logits/rejected": -4.509836196899414,
       "logps/chosen": -274.000732421875,
       "logps/rejected": -205.8054962158203,
-      "loss": 0.6931,
+      "loss": 0.6932,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,586 +24,586 @@
       "step": 1
     },
     {
-      "epoch": 0.02857142857142857,
-      "grad_norm": 1668.215158817105,
-      "learning_rate": 4.9899357349880975e-08,
-      "logits/chosen": -4.21472692489624,
-      "logits/rejected": -4.488520622253418,
-      "logps/chosen": -318.4333801269531,
-      "logps/rejected": -257.2440185546875,
-      "loss": 0.6858,
-      "rewards/accuracies": 0.4930555522441864,
-      "rewards/chosen": 0.03888694569468498,
-      "rewards/margins": 0.020132217556238174,
-      "rewards/rejected": 0.018754728138446808,
+      "epoch": 0.03,
+      "grad_norm": 3122.252138846549,
+      "learning_rate": 1.4285714285714285e-07,
+      "logits/chosen": -4.2117600440979,
+      "logits/rejected": -4.4855546951293945,
+      "logps/chosen": -318.3944396972656,
+      "logps/rejected": -257.1120910644531,
+      "loss": 0.7578,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.11668112874031067,
+      "rewards/margins": -0.05277401953935623,
+      "rewards/rejected": 0.1694551408290863,
       "step": 10
     },
     {
-      "epoch": 0.05714285714285714,
-      "grad_norm": 1345.3140295690048,
-      "learning_rate": 4.959823971496574e-08,
-      "logits/chosen": -4.247828006744385,
-      "logits/rejected": -4.502226829528809,
-      "logps/chosen": -304.4950866699219,
-      "logps/rejected": -244.0281219482422,
-      "loss": 0.6107,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.37075769901275635,
-      "rewards/margins": 0.27177533507347107,
-      "rewards/rejected": 0.09898237138986588,
+      "epoch": 0.06,
+      "grad_norm": 2683.55929188042,
+      "learning_rate": 2.857142857142857e-07,
+      "logits/chosen": -4.264363765716553,
+      "logits/rejected": -4.5196099281311035,
+      "logps/chosen": -303.1786193847656,
+      "logps/rejected": -243.7255096435547,
+      "loss": 0.5287,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 2.058025598526001,
+      "rewards/margins": 1.5574162006378174,
+      "rewards/rejected": 0.5006095767021179,
       "step": 20
     },
     {
-      "epoch": 0.08571428571428572,
-      "grad_norm": 1125.2250514399805,
-      "learning_rate": 4.9099071517396326e-08,
-      "logits/chosen": -4.306983947753906,
-      "logits/rejected": -4.5679826736450195,
-      "logps/chosen": -304.89337158203125,
-      "logps/rejected": -258.8996887207031,
-      "loss": 0.5148,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 0.8106807470321655,
-      "rewards/margins": 0.5930811762809753,
-      "rewards/rejected": 0.21759963035583496,
+      "epoch": 0.09,
+      "grad_norm": 1727.068886617789,
+      "learning_rate": 4.285714285714285e-07,
+      "logits/chosen": -4.307942867279053,
+      "logits/rejected": -4.567526340484619,
+      "logps/chosen": -299.24615478515625,
+      "logps/rejected": -256.9350280761719,
+      "loss": 0.4422,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 7.268563747406006,
+      "rewards/margins": 4.868700981140137,
+      "rewards/rejected": 2.3998632431030273,
       "step": 30
     },
     {
-      "epoch": 0.11428571428571428,
-      "grad_norm": 1094.2742476604449,
-      "learning_rate": 4.8405871765993426e-08,
-      "logits/chosen": -4.310123443603516,
-      "logits/rejected": -4.577700614929199,
-      "logps/chosen": -292.9500427246094,
-      "logps/rejected": -234.384765625,
-      "loss": 0.4433,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 1.2921748161315918,
-      "rewards/margins": 0.8818836212158203,
-      "rewards/rejected": 0.41029104590415955,
+      "epoch": 0.11,
+      "grad_norm": 2206.909408534419,
+      "learning_rate": 4.996892303047305e-07,
+      "logits/chosen": -4.312300682067871,
+      "logits/rejected": -4.578764915466309,
+      "logps/chosen": -288.2650451660156,
+      "logps/rejected": -235.3504638671875,
+      "loss": 0.5344,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 7.26935338973999,
+      "rewards/margins": 7.414456367492676,
+      "rewards/rejected": -0.145103320479393,
       "step": 40
     },
     {
-      "epoch": 0.14285714285714285,
-      "grad_norm": 866.0178113474443,
-      "learning_rate": 4.7524221697560474e-08,
-      "logits/chosen": -4.307633876800537,
-      "logits/rejected": -4.553541660308838,
-      "logps/chosen": -298.9430847167969,
-      "logps/rejected": -252.4582977294922,
-      "loss": 0.3906,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 1.8912757635116577,
-      "rewards/margins": 1.240208387374878,
-      "rewards/rejected": 0.6510674357414246,
+      "epoch": 0.14,
+      "grad_norm": 1574.7884185445766,
+      "learning_rate": 4.972077065562821e-07,
+      "logits/chosen": -4.287051200866699,
+      "logits/rejected": -4.532064914703369,
+      "logps/chosen": -295.8678894042969,
+      "logps/rejected": -256.9671325683594,
+      "loss": 0.514,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 6.857720851898193,
+      "rewards/margins": 10.064432144165039,
+      "rewards/rejected": -3.206712007522583,
       "step": 50
     },
     {
-      "epoch": 0.17142857142857143,
-      "grad_norm": 837.0640990054532,
-      "learning_rate": 4.646121984004665e-08,
-      "logits/chosen": -4.309387683868408,
-      "logits/rejected": -4.5390214920043945,
-      "logps/chosen": -307.09820556640625,
-      "logps/rejected": -260.90191650390625,
-      "loss": 0.3814,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 2.126704216003418,
-      "rewards/margins": 1.2716383934020996,
-      "rewards/rejected": 0.8550659418106079,
+      "epoch": 0.17,
+      "grad_norm": 2086.3490202582466,
+      "learning_rate": 4.922693215572695e-07,
+      "logits/chosen": -4.274147033691406,
+      "logits/rejected": -4.500526428222656,
+      "logps/chosen": -304.9868469238281,
+      "logps/rejected": -266.1625061035156,
+      "loss": 0.5936,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 6.364766597747803,
+      "rewards/margins": 9.915193557739258,
+      "rewards/rejected": -3.550427198410034,
       "step": 60
     },
     {
       "epoch": 0.2,
-      "grad_norm": 985.5848045316892,
-      "learning_rate": 4.522542485937369e-08,
-      "logits/chosen": -4.4262213706970215,
-      "logits/rejected": -4.558186054229736,
-      "logps/chosen": -284.1617736816406,
-      "logps/rejected": -235.8411102294922,
-      "loss": 0.3695,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 2.3711953163146973,
-      "rewards/margins": 1.6806213855743408,
-      "rewards/rejected": 0.690574049949646,
+      "grad_norm": 2489.785834029294,
+      "learning_rate": 4.849231551964771e-07,
+      "logits/chosen": -4.388774871826172,
+      "logits/rejected": -4.523660659790039,
+      "logps/chosen": -281.50799560546875,
+      "logps/rejected": -240.20126342773438,
+      "loss": 0.578,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 7.396153926849365,
+      "rewards/margins": 10.37517261505127,
+      "rewards/rejected": -2.9790191650390625,
       "step": 70
     },
     {
-      "epoch": 0.22857142857142856,
-      "grad_norm": 951.7016793785169,
-      "learning_rate": 4.3826786650090273e-08,
-      "logits/chosen": -4.280414581298828,
-      "logits/rejected": -4.534539222717285,
-      "logps/chosen": -290.68597412109375,
-      "logps/rejected": -239.3174591064453,
-      "loss": 0.3466,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 2.500866413116455,
-      "rewards/margins": 1.933850646018982,
-      "rewards/rejected": 0.5670154094696045,
+      "epoch": 0.23,
+      "grad_norm": 1795.0951767218703,
+      "learning_rate": 4.752422169756047e-07,
+      "logits/chosen": -4.22324275970459,
+      "logits/rejected": -4.482357025146484,
+      "logps/chosen": -289.7461853027344,
+      "logps/rejected": -244.2007293701172,
+      "loss": 0.6974,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 5.9415082931518555,
+      "rewards/margins": 9.690756797790527,
+      "rewards/rejected": -3.749248504638672,
       "step": 80
     },
     {
-      "epoch": 0.2571428571428571,
-      "grad_norm": 868.4695107443264,
-      "learning_rate": 4.2276566224671614e-08,
-      "logits/chosen": -4.205895900726318,
-      "logits/rejected": -4.440802574157715,
-      "logps/chosen": -301.97674560546875,
-      "logps/rejected": -257.63153076171875,
-      "loss": 0.3457,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 2.7932560443878174,
-      "rewards/margins": 1.9473575353622437,
-      "rewards/rejected": 0.8458986282348633,
+      "epoch": 0.26,
+      "grad_norm": 1863.4883740900614,
+      "learning_rate": 4.6332272040803887e-07,
+      "logits/chosen": -4.147845268249512,
+      "logits/rejected": -4.379548072814941,
+      "logps/chosen": -301.5574645996094,
+      "logps/rejected": -264.04815673828125,
+      "loss": 0.6584,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 6.005797386169434,
+      "rewards/margins": 10.730647087097168,
+      "rewards/rejected": -4.724849700927734,
       "step": 90
     },
     {
-      "epoch": 0.2857142857142857,
-      "grad_norm": 1324.6980610698608,
-      "learning_rate": 4.058724504646834e-08,
-      "logits/chosen": -4.310187816619873,
-      "logits/rejected": -4.531655311584473,
-      "logps/chosen": -290.22113037109375,
-      "logps/rejected": -240.4936981201172,
-      "loss": 0.3253,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 2.803382396697998,
-      "rewards/margins": 1.9496362209320068,
-      "rewards/rejected": 0.8537459373474121,
+      "epoch": 0.29,
+      "grad_norm": 2150.0529520665987,
+      "learning_rate": 4.492831268057306e-07,
+      "logits/chosen": -4.204574108123779,
+      "logits/rejected": -4.426244735717773,
+      "logps/chosen": -287.6278076171875,
+      "logps/rejected": -244.3843231201172,
+      "loss": 0.7723,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 8.200125694274902,
+      "rewards/margins": 10.383246421813965,
+      "rewards/rejected": -2.1831212043762207,
       "step": 100
     },
     {
-      "epoch": 0.2857142857142857,
-      "eval_logits/chosen": -3.2365729808807373,
-      "eval_logits/rejected": -3.2365729808807373,
-      "eval_logps/chosen": -158.97109985351562,
-      "eval_logps/rejected": -158.97109985351562,
+      "epoch": 0.29,
+      "eval_logits/chosen": -3.147157669067383,
+      "eval_logits/rejected": -3.147157669067383,
+      "eval_logps/chosen": -157.6016845703125,
+      "eval_logps/rejected": -157.6016845703125,
       "eval_loss": 0.6931471824645996,
       "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": -1.697042465209961,
+      "eval_rewards/chosen": -2.0246658325195312,
       "eval_rewards/margins": 0.0,
-      "eval_rewards/rejected": -1.697042465209961,
-      "eval_runtime": 1.5274,
-      "eval_samples_per_second": 0.655,
-      "eval_steps_per_second": 0.655,
+      "eval_rewards/rejected": -2.0246658325195312,
+      "eval_runtime": 1.5111,
+      "eval_samples_per_second": 0.662,
+      "eval_steps_per_second": 0.662,
       "step": 100
     },
     {
-      "epoch": 0.3142857142857143,
-      "grad_norm": 904.1535779728154,
-      "learning_rate": 3.8772424536302564e-08,
-      "logits/chosen": -4.3289313316345215,
-      "logits/rejected": -4.571288108825684,
-      "logps/chosen": -297.5599060058594,
-      "logps/rejected": -249.8257293701172,
-      "loss": 0.345,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 2.935572385787964,
-      "rewards/margins": 2.1961002349853516,
-      "rewards/rejected": 0.7394723892211914,
+      "epoch": 0.31,
+      "grad_norm": 2069.7674123460392,
+      "learning_rate": 4.332629679574565e-07,
+      "logits/chosen": -4.301981449127197,
+      "logits/rejected": -4.562737464904785,
+      "logps/chosen": -298.0445861816406,
+      "logps/rejected": -258.07379150390625,
+      "loss": 0.9782,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 5.386460304260254,
+      "rewards/margins": 12.155603408813477,
+      "rewards/rejected": -6.769143581390381,
       "step": 110
     },
     {
-      "epoch": 0.34285714285714286,
-      "grad_norm": 1086.0405455439977,
-      "learning_rate": 3.6846716561824964e-08,
-      "logits/chosen": -4.373476982116699,
-      "logits/rejected": -4.619527816772461,
-      "logps/chosen": -287.13385009765625,
-      "logps/rejected": -237.6899871826172,
-      "loss": 0.3263,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 2.900177478790283,
-      "rewards/margins": 2.18641996383667,
-      "rewards/rejected": 0.7137576937675476,
+      "epoch": 0.34,
+      "grad_norm": 2493.1141420840436,
+      "learning_rate": 4.154214593992149e-07,
+      "logits/chosen": -4.2862958908081055,
+      "logits/rejected": -4.542973518371582,
+      "logps/chosen": -285.873046875,
+      "logps/rejected": -243.8842010498047,
+      "loss": 0.9519,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 7.061153411865234,
+      "rewards/margins": 11.827821731567383,
+      "rewards/rejected": -4.766669273376465,
       "step": 120
     },
     {
-      "epoch": 0.37142857142857144,
-      "grad_norm": 753.3076911412429,
-      "learning_rate": 3.482562579134809e-08,
-      "logits/chosen": -4.375483512878418,
-      "logits/rejected": -4.624612331390381,
-      "logps/chosen": -276.9853515625,
-      "logps/rejected": -218.39633178710938,
-      "loss": 0.3228,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 2.8573689460754395,
-      "rewards/margins": 2.0853238105773926,
-      "rewards/rejected": 0.7720457315444946,
+      "epoch": 0.37,
+      "grad_norm": 1751.0600688695415,
+      "learning_rate": 3.959359180586975e-07,
+      "logits/chosen": -4.280123710632324,
+      "logits/rejected": -4.522739887237549,
+      "logps/chosen": -277.5672302246094,
+      "logps/rejected": -226.9970703125,
+      "loss": 0.7208,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 5.132817268371582,
+      "rewards/margins": 12.189440727233887,
+      "rewards/rejected": -7.056623935699463,
       "step": 130
     },
     {
       "epoch": 0.4,
-      "grad_norm": 840.6908677183937,
-      "learning_rate": 3.272542485937369e-08,
-      "logits/chosen": -4.290375232696533,
-      "logits/rejected": -4.607089996337891,
-      "logps/chosen": -294.1933288574219,
-      "logps/rejected": -239.8271942138672,
-      "loss": 0.3428,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 2.845167875289917,
-      "rewards/margins": 2.3569939136505127,
-      "rewards/rejected": 0.48817411065101624,
+      "grad_norm": 1830.8828386105183,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": -4.162067413330078,
+      "logits/rejected": -4.473877429962158,
+      "logps/chosen": -294.2828369140625,
+      "logps/rejected": -248.1217041015625,
+      "loss": 0.8272,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 5.600853443145752,
+      "rewards/margins": 12.919031143188477,
+      "rewards/rejected": -7.31817626953125,
       "step": 140
     },
     {
-      "epoch": 0.42857142857142855,
-      "grad_norm": 1010.0880176318159,
-      "learning_rate": 3.056302334890786e-08,
-      "logits/chosen": -4.25304651260376,
-      "logits/rejected": -4.517129421234131,
-      "logps/chosen": -293.35797119140625,
-      "logps/rejected": -250.48641967773438,
-      "loss": 0.313,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.026089906692505,
-      "rewards/margins": 2.4186887741088867,
-      "rewards/rejected": 0.6074013113975525,
+      "epoch": 0.43,
+      "grad_norm": 1975.8832437780866,
+      "learning_rate": 3.528217757826529e-07,
+      "logits/chosen": -4.179337978363037,
+      "logits/rejected": -4.442940711975098,
+      "logps/chosen": -295.30914306640625,
+      "logps/rejected": -259.9874267578125,
+      "loss": 0.8642,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 4.101029872894287,
+      "rewards/margins": 12.387203216552734,
+      "rewards/rejected": -8.286172866821289,
       "step": 150
     },
     {
-      "epoch": 0.45714285714285713,
-      "grad_norm": 881.3705397776472,
-      "learning_rate": 2.8355831645441387e-08,
-      "logits/chosen": -4.279029369354248,
-      "logits/rejected": -4.569698810577393,
-      "logps/chosen": -294.6650695800781,
-      "logps/rejected": -235.29067993164062,
-      "loss": 0.338,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.13142728805542,
-      "rewards/margins": 2.386794090270996,
-      "rewards/rejected": 0.7446335554122925,
+      "epoch": 0.46,
+      "grad_norm": 1974.3387943820865,
+      "learning_rate": 3.296216625629211e-07,
+      "logits/chosen": -4.1478681564331055,
+      "logits/rejected": -4.431545257568359,
+      "logps/chosen": -294.5439758300781,
+      "logps/rejected": -244.33657836914062,
+      "loss": 0.9334,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 6.383957386016846,
+      "rewards/margins": 13.940587997436523,
+      "rewards/rejected": -7.556630611419678,
       "step": 160
     },
     {
-      "epoch": 0.4857142857142857,
-      "grad_norm": 857.1741181734847,
-      "learning_rate": 2.6121620758762875e-08,
-      "logits/chosen": -4.228262901306152,
-      "logits/rejected": -4.463376522064209,
-      "logps/chosen": -294.2881774902344,
-      "logps/rejected": -241.1619415283203,
-      "loss": 0.3264,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 3.1862380504608154,
-      "rewards/margins": 2.542022228240967,
-      "rewards/rejected": 0.6442161202430725,
+      "epoch": 0.49,
+      "grad_norm": 1732.7845040237094,
+      "learning_rate": 3.056302334890786e-07,
+      "logits/chosen": -4.18727970123291,
+      "logits/rejected": -4.425799369812012,
+      "logps/chosen": -294.9515075683594,
+      "logps/rejected": -250.8203125,
+      "loss": 0.8554,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 5.709182262420654,
+      "rewards/margins": 14.079116821289062,
+      "rewards/rejected": -8.369935989379883,
       "step": 170
     },
     {
-      "epoch": 0.5142857142857142,
-      "grad_norm": 923.726950031985,
-      "learning_rate": 2.3878379241237133e-08,
-      "logits/chosen": -4.36321496963501,
-      "logits/rejected": -4.595080375671387,
-      "logps/chosen": -283.4067687988281,
-      "logps/rejected": -241.1226043701172,
-      "loss": 0.3098,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 3.2351880073547363,
-      "rewards/margins": 2.5559356212615967,
-      "rewards/rejected": 0.679252028465271,
+      "epoch": 0.51,
+      "grad_norm": 2121.8411757635613,
+      "learning_rate": 2.810859261618713e-07,
+      "logits/chosen": -4.313704490661621,
+      "logits/rejected": -4.544769287109375,
+      "logps/chosen": -282.9936828613281,
+      "logps/rejected": -250.189453125,
+      "loss": 0.7998,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 6.883467674255371,
+      "rewards/margins": 14.591836929321289,
+      "rewards/rejected": -7.708369255065918,
       "step": 180
     },
     {
-      "epoch": 0.5428571428571428,
-      "grad_norm": 792.8076625116224,
-      "learning_rate": 2.164416835455862e-08,
-      "logits/chosen": -4.323281288146973,
-      "logits/rejected": -4.492599964141846,
-      "logps/chosen": -305.88592529296875,
-      "logps/rejected": -257.3919982910156,
-      "loss": 0.2759,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 3.2042744159698486,
-      "rewards/margins": 2.557298421859741,
-      "rewards/rejected": 0.6469758152961731,
+      "epoch": 0.54,
+      "grad_norm": 1756.6999736537189,
+      "learning_rate": 2.5623267293451823e-07,
+      "logits/chosen": -4.283580303192139,
+      "logits/rejected": -4.457066535949707,
+      "logps/chosen": -305.109375,
+      "logps/rejected": -265.3091125488281,
+      "loss": 0.6833,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 7.185091495513916,
+      "rewards/margins": 13.808235168457031,
+      "rewards/rejected": -6.623143196105957,
       "step": 190
     },
     {
-      "epoch": 0.5714285714285714,
-      "grad_norm": 757.2657183486399,
-      "learning_rate": 1.943697665109214e-08,
-      "logits/chosen": -4.354622840881348,
-      "logits/rejected": -4.594660758972168,
-      "logps/chosen": -290.7183532714844,
-      "logps/rejected": -249.0703582763672,
-      "loss": 0.3071,
+      "epoch": 0.57,
+      "grad_norm": 1748.1359792805438,
+      "learning_rate": 2.3131747660339394e-07,
+      "logits/chosen": -4.295716285705566,
+      "logits/rejected": -4.545838832855225,
+      "logps/chosen": -289.03521728515625,
+      "logps/rejected": -257.3003234863281,
+      "loss": 0.7717,
       "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 3.2066097259521484,
-      "rewards/margins": 2.334256649017334,
-      "rewards/rejected": 0.8723530769348145,
+      "rewards/chosen": 8.096379280090332,
+      "rewards/margins": 14.581645011901855,
+      "rewards/rejected": -6.485265254974365,
       "step": 200
     },
     {
-      "epoch": 0.5714285714285714,
-      "eval_logits/chosen": -3.2287425994873047,
-      "eval_logits/rejected": -3.2287425994873047,
-      "eval_logps/chosen": -159.75973510742188,
-      "eval_logps/rejected": -159.75973510742188,
+      "epoch": 0.57,
+      "eval_logits/chosen": -3.163522243499756,
+      "eval_logits/rejected": -3.163522243499756,
+      "eval_logps/chosen": -167.66006469726562,
+      "eval_logps/rejected": -167.66006469726562,
       "eval_loss": 0.6931471824645996,
       "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": -2.0913619995117188,
+      "eval_rewards/chosen": -12.083049774169922,
       "eval_rewards/margins": 0.0,
-      "eval_rewards/rejected": -2.0913619995117188,
-      "eval_runtime": 1.503,
-      "eval_samples_per_second": 0.665,
-      "eval_steps_per_second": 0.665,
+      "eval_rewards/rejected": -12.083049774169922,
+      "eval_runtime": 1.4711,
+      "eval_samples_per_second": 0.68,
+      "eval_steps_per_second": 0.68,
       "step": 200
     },
     {
       "epoch": 0.6,
-      "grad_norm": 891.2229023091579,
-      "learning_rate": 1.7274575140626317e-08,
-      "logits/chosen": -4.2898783683776855,
-      "logits/rejected": -4.5833001136779785,
-      "logps/chosen": -304.8259582519531,
-      "logps/rejected": -254.60757446289062,
-      "loss": 0.3024,
+      "grad_norm": 1829.602720191424,
+      "learning_rate": 2.065879555832674e-07,
+      "logits/chosen": -4.250877857208252,
+      "logits/rejected": -4.542287826538086,
+      "logps/chosen": -303.2696838378906,
+      "logps/rejected": -262.7691650390625,
+      "loss": 0.8229,
       "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 3.1973044872283936,
-      "rewards/margins": 2.722479820251465,
-      "rewards/rejected": 0.47482460737228394,
+      "rewards/chosen": 7.95089864730835,
+      "rewards/margins": 15.162821769714355,
+      "rewards/rejected": -7.211922645568848,
       "step": 210
     },
     {
-      "epoch": 0.6285714285714286,
-      "grad_norm": 938.9689090425716,
-      "learning_rate": 1.517437420865191e-08,
-      "logits/chosen": -4.243287086486816,
-      "logits/rejected": -4.58882999420166,
-      "logps/chosen": -294.97943115234375,
-      "logps/rejected": -225.01730346679688,
-      "loss": 0.2916,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 3.2672336101531982,
-      "rewards/margins": 2.979445695877075,
-      "rewards/rejected": 0.2877880036830902,
+      "epoch": 0.63,
+      "grad_norm": 1788.0044780541311,
+      "learning_rate": 1.8228988296424875e-07,
+      "logits/chosen": -4.257784366607666,
+      "logits/rejected": -4.608870983123779,
+      "logps/chosen": -293.54071044921875,
+      "logps/rejected": -233.70166015625,
+      "loss": 0.7993,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 7.973156929016113,
+      "rewards/margins": 16.081945419311523,
+      "rewards/rejected": -8.108789443969727,
       "step": 220
     },
     {
-      "epoch": 0.6571428571428571,
-      "grad_norm": 889.4233945739674,
-      "learning_rate": 1.3153283438175034e-08,
-      "logits/chosen": -4.367541313171387,
-      "logits/rejected": -4.557114601135254,
-      "logps/chosen": -279.52362060546875,
-      "logps/rejected": -226.0491180419922,
-      "loss": 0.2804,
+      "epoch": 0.66,
+      "grad_norm": 1900.3878505004946,
+      "learning_rate": 1.5866474390840124e-07,
+      "logits/chosen": -4.334306716918945,
+      "logits/rejected": -4.525221824645996,
+      "logps/chosen": -278.41827392578125,
+      "logps/rejected": -233.2617645263672,
+      "loss": 0.6293,
       "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 2.717285633087158,
-      "rewards/margins": 2.2621235847473145,
-      "rewards/rejected": 0.45516151189804077,
+      "rewards/chosen": 6.539956569671631,
+      "rewards/margins": 12.842289924621582,
+      "rewards/rejected": -6.30233097076416,
       "step": 230
     },
     {
-      "epoch": 0.6857142857142857,
-      "grad_norm": 764.4075642697645,
-      "learning_rate": 1.1227575463697438e-08,
-      "logits/chosen": -4.391533851623535,
-      "logits/rejected": -4.709025859832764,
-      "logps/chosen": -256.52532958984375,
-      "logps/rejected": -215.2234649658203,
-      "loss": 0.287,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 2.9945430755615234,
-      "rewards/margins": 2.5624358654022217,
-      "rewards/rejected": 0.4321066439151764,
+      "epoch": 0.69,
+      "grad_norm": 1759.0935765619065,
+      "learning_rate": 1.3594733566170925e-07,
+      "logits/chosen": -4.360232353210449,
+      "logits/rejected": -4.688153266906738,
+      "logps/chosen": -255.55807495117188,
+      "logps/rejected": -222.76504516601562,
+      "loss": 1.0017,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 6.956341743469238,
+      "rewards/margins": 13.633699417114258,
+      "rewards/rejected": -6.677358150482178,
       "step": 240
     },
     {
-      "epoch": 0.7142857142857143,
-      "grad_norm": 1078.8910476393369,
-      "learning_rate": 9.412754953531663e-09,
-      "logits/chosen": -4.338193416595459,
-      "logits/rejected": -4.612923622131348,
-      "logps/chosen": -276.8057861328125,
-      "logps/rejected": -232.43807983398438,
-      "loss": 0.2923,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 2.8848717212677,
-      "rewards/margins": 2.400455951690674,
-      "rewards/rejected": 0.4844156801700592,
+      "epoch": 0.71,
+      "grad_norm": 1730.0292892661773,
+      "learning_rate": 1.1436343403356016e-07,
+      "logits/chosen": -4.352273941040039,
+      "logits/rejected": -4.6365180015563965,
+      "logps/chosen": -276.07659912109375,
+      "logps/rejected": -241.01425170898438,
+      "loss": 0.8087,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 6.498913764953613,
+      "rewards/margins": 14.106257438659668,
+      "rewards/rejected": -7.6073455810546875,
       "step": 250
     },
     {
-      "epoch": 0.7428571428571429,
-      "grad_norm": 888.6230207618648,
-      "learning_rate": 7.723433775328384e-09,
-      "logits/chosen": -4.384097099304199,
-      "logits/rejected": -4.630164623260498,
-      "logps/chosen": -269.947265625,
-      "logps/rejected": -240.2582244873047,
-      "loss": 0.2917,
+      "epoch": 0.74,
+      "grad_norm": 1731.9100006560693,
+      "learning_rate": 9.412754953531663e-08,
+      "logits/chosen": -4.388053894042969,
+      "logits/rejected": -4.64432954788208,
+      "logps/chosen": -270.12225341796875,
+      "logps/rejected": -249.11416625976562,
+      "loss": 0.6771,
       "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 2.7555510997772217,
-      "rewards/margins": 2.3154397010803223,
-      "rewards/rejected": 0.4401116967201233,
+      "rewards/chosen": 5.336110591888428,
+      "rewards/margins": 13.311798095703125,
+      "rewards/rejected": -7.975686073303223,
       "step": 260
     },
     {
-      "epoch": 0.7714285714285715,
-      "grad_norm": 839.6114448931011,
-      "learning_rate": 6.173213349909728e-09,
-      "logits/chosen": -4.516074180603027,
-      "logits/rejected": -4.6834282875061035,
-      "logps/chosen": -271.71820068359375,
-      "logps/rejected": -227.9601287841797,
-      "loss": 0.3199,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 2.6797890663146973,
-      "rewards/margins": 2.0464982986450195,
-      "rewards/rejected": 0.6332908868789673,
+      "epoch": 0.77,
+      "grad_norm": 1559.894108247427,
+      "learning_rate": 7.544079547848181e-08,
+      "logits/chosen": -4.511970520019531,
+      "logits/rejected": -4.677350044250488,
+      "logps/chosen": -272.5389709472656,
+      "logps/rejected": -237.0705108642578,
+      "loss": 0.816,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 4.5388288497924805,
+      "rewards/margins": 12.382614135742188,
+      "rewards/rejected": -7.843785762786865,
       "step": 270
     },
     {
       "epoch": 0.8,
-      "grad_norm": 794.9828383467956,
-      "learning_rate": 4.7745751406263165e-09,
-      "logits/chosen": -4.297934532165527,
-      "logits/rejected": -4.586764335632324,
-      "logps/chosen": -272.8429870605469,
-      "logps/rejected": -229.56021118164062,
-      "loss": 0.2842,
-      "rewards/accuracies": 0.893750011920929,
-      "rewards/chosen": 3.1210646629333496,
-      "rewards/margins": 2.520183801651001,
-      "rewards/rejected": 0.6008811593055725,
+      "grad_norm": 2131.975353118901,
+      "learning_rate": 5.848888922025552e-08,
+      "logits/chosen": -4.293630123138428,
+      "logits/rejected": -4.587409496307373,
+      "logps/chosen": -272.18914794921875,
+      "logps/rejected": -237.04507446289062,
+      "loss": 0.7076,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 6.89593505859375,
+      "rewards/margins": 13.17906379699707,
+      "rewards/rejected": -6.283128261566162,
       "step": 280
     },
     {
-      "epoch": 0.8285714285714286,
-      "grad_norm": 1122.8170349728516,
-      "learning_rate": 3.5387801599533474e-09,
-      "logits/chosen": -4.3172149658203125,
-      "logits/rejected": -4.5055928230285645,
-      "logps/chosen": -280.2183837890625,
-      "logps/rejected": -236.1828155517578,
-      "loss": 0.3083,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 3.173701286315918,
-      "rewards/margins": 2.5917415618896484,
-      "rewards/rejected": 0.5819598436355591,
+      "epoch": 0.83,
+      "grad_norm": 2073.5868352302195,
+      "learning_rate": 4.3440306421001324e-08,
+      "logits/chosen": -4.306157112121582,
+      "logits/rejected": -4.501837253570557,
+      "logps/chosen": -279.41571044921875,
+      "logps/rejected": -245.3014678955078,
+      "loss": 0.7257,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 7.150078773498535,
+      "rewards/margins": 15.104804992675781,
+      "rewards/rejected": -7.954724311828613,
       "step": 290
     },
     {
-      "epoch": 0.8571428571428571,
-      "grad_norm": 746.1081306618772,
-      "learning_rate": 2.475778302439524e-09,
-      "logits/chosen": -4.29483699798584,
-      "logits/rejected": -4.537248134613037,
-      "logps/chosen": -296.06756591796875,
-      "logps/rejected": -239.85092163085938,
-      "loss": 0.3336,
+      "epoch": 0.86,
+      "grad_norm": 1664.7949029760775,
+      "learning_rate": 3.044460665744283e-08,
+      "logits/chosen": -4.291565418243408,
+      "logits/rejected": -4.546942234039307,
+      "logps/chosen": -294.171142578125,
+      "logps/rejected": -248.09219360351562,
+      "loss": 0.782,
       "rewards/accuracies": 0.875,
-      "rewards/chosen": 3.3819689750671387,
-      "rewards/margins": 2.6246695518493652,
-      "rewards/rejected": 0.7572996020317078,
+      "rewards/chosen": 8.660406112670898,
+      "rewards/margins": 15.387075424194336,
+      "rewards/rejected": -6.726672172546387,
       "step": 300
     },
     {
-      "epoch": 0.8571428571428571,
-      "eval_logits/chosen": -3.2277979850769043,
-      "eval_logits/rejected": -3.2277979850769043,
-      "eval_logps/chosen": -158.84542846679688,
-      "eval_logps/rejected": -158.84542846679688,
+      "epoch": 0.86,
+      "eval_logits/chosen": -3.222372531890869,
+      "eval_logits/rejected": -3.222372531890869,
+      "eval_logps/chosen": -164.06509399414062,
+      "eval_logps/rejected": -164.06509399414062,
       "eval_loss": 0.6931471824645996,
       "eval_rewards/accuracies": 0.0,
-      "eval_rewards/chosen": -1.6342105865478516,
+      "eval_rewards/chosen": -8.488082885742188,
       "eval_rewards/margins": 0.0,
-      "eval_rewards/rejected": -1.6342105865478516,
-      "eval_runtime": 1.5181,
-      "eval_samples_per_second": 0.659,
-      "eval_steps_per_second": 0.659,
+      "eval_rewards/rejected": -8.488082885742188,
+      "eval_runtime": 1.4741,
+      "eval_samples_per_second": 0.678,
+      "eval_steps_per_second": 0.678,
       "step": 300
     },
     {
-      "epoch": 0.8857142857142857,
-      "grad_norm": 1352.5130710151727,
-      "learning_rate": 1.5941282340065698e-09,
-      "logits/chosen": -4.436648368835449,
-      "logits/rejected": -4.576234817504883,
-      "logps/chosen": -260.4552001953125,
-      "logps/rejected": -226.2211456298828,
-      "loss": 0.2894,
+      "epoch": 0.89,
+      "grad_norm": 2673.859330983399,
+      "learning_rate": 1.9630947032398066e-08,
+      "logits/chosen": -4.44521427154541,
+      "logits/rejected": -4.5893964767456055,
+      "logps/chosen": -260.0516662597656,
+      "logps/rejected": -234.23690795898438,
+      "loss": 0.7257,
       "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 2.8304812908172607,
-      "rewards/margins": 2.2945332527160645,
-      "rewards/rejected": 0.535947859287262,
+      "rewards/chosen": 6.064515113830566,
+      "rewards/margins": 13.008413314819336,
+      "rewards/rejected": -6.943899631500244,
       "step": 310
     },
     {
-      "epoch": 0.9142857142857143,
-      "grad_norm": 759.0330474241811,
-      "learning_rate": 9.009284826036689e-10,
-      "logits/chosen": -4.273808002471924,
-      "logits/rejected": -4.527020454406738,
-      "logps/chosen": -290.4542541503906,
-      "logps/rejected": -243.5283966064453,
-      "loss": 0.314,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 3.2147815227508545,
-      "rewards/margins": 2.507025718688965,
-      "rewards/rejected": 0.7077558040618896,
+      "epoch": 0.91,
+      "grad_norm": 1691.3211570500287,
+      "learning_rate": 1.1106798553464802e-08,
+      "logits/chosen": -4.273613929748535,
+      "logits/rejected": -4.540968894958496,
+      "logps/chosen": -289.5185546875,
+      "logps/rejected": -251.7852325439453,
+      "loss": 0.6936,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 7.365248203277588,
+      "rewards/margins": 14.206552505493164,
+      "rewards/rejected": -6.84130334854126,
       "step": 320
     },
     {
-      "epoch": 0.9428571428571428,
-      "grad_norm": 1012.631208038228,
-      "learning_rate": 4.017602850342583e-10,
-      "logits/chosen": -4.329623222351074,
-      "logits/rejected": -4.535677433013916,
-      "logps/chosen": -303.55926513671875,
-      "logps/rejected": -251.89437866210938,
-      "loss": 0.3017,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": 3.013214588165283,
-      "rewards/margins": 2.4605515003204346,
-      "rewards/rejected": 0.5526631474494934,
+      "epoch": 0.94,
+      "grad_norm": 2384.133770310185,
+      "learning_rate": 4.956878037864043e-09,
+      "logits/chosen": -4.335446834564209,
+      "logits/rejected": -4.543330669403076,
+      "logps/chosen": -303.6029968261719,
+      "logps/rejected": -260.98760986328125,
+      "loss": 0.7573,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 5.982678413391113,
+      "rewards/margins": 13.970565795898438,
+      "rewards/rejected": -7.987887382507324,
       "step": 330
     },
     {
-      "epoch": 0.9714285714285714,
-      "grad_norm": 1047.7085022489064,
-      "learning_rate": 1.0064265011902328e-10,
-      "logits/chosen": -4.3172688484191895,
-      "logits/rejected": -4.609848976135254,
-      "logps/chosen": -283.23883056640625,
-      "logps/rejected": -227.2130126953125,
-      "loss": 0.3029,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 2.853224277496338,
-      "rewards/margins": 2.2073826789855957,
-      "rewards/rejected": 0.6458419561386108,
+      "epoch": 0.97,
+      "grad_norm": 2211.192973189294,
+      "learning_rate": 1.2423061586496476e-09,
+      "logits/chosen": -4.319240093231201,
+      "logits/rejected": -4.623973846435547,
+      "logps/chosen": -283.38006591796875,
+      "logps/rejected": -234.4169921875,
+      "loss": 0.8286,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 5.565188407897949,
+      "rewards/margins": 11.47750186920166,
+      "rewards/rejected": -5.912313938140869,
       "step": 340
     },
     {
       "epoch": 1.0,
-      "grad_norm": 570.6961447414922,
+      "grad_norm": 1586.2152634156816,
       "learning_rate": 0.0,
-      "logits/chosen": -4.2885661125183105,
-      "logits/rejected": -4.4927144050598145,
-      "logps/chosen": -289.79022216796875,
-      "logps/rejected": -244.1138916015625,
-      "loss": 0.2904,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": 3.115464448928833,
-      "rewards/margins": 2.5564632415771484,
-      "rewards/rejected": 0.5590011477470398,
+      "logits/chosen": -4.285008430480957,
+      "logits/rejected": -4.503040790557861,
+      "logps/chosen": -289.0204162597656,
+      "logps/rejected": -251.9452362060547,
+      "loss": 0.7666,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 7.000736236572266,
+      "rewards/margins": 13.714078903198242,
+      "rewards/rejected": -6.713343143463135,
       "step": 350
     },
     {
       "epoch": 1.0,
       "step": 350,
       "total_flos": 0.0,
-      "train_loss": 0.34604193687438967,
-      "train_runtime": 5793.0735,
-      "train_samples_per_second": 7.726,
-      "train_steps_per_second": 0.06
+      "train_loss": 0.7428068714482444,
+      "train_runtime": 5299.044,
+      "train_samples_per_second": 8.446,
+      "train_steps_per_second": 0.066
     }
   ],
   "logging_steps": 10,
@@ -611,18 +611,6 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,
-  "stateful_callbacks": {
-    "TrainerControl": {
-      "args": {
-        "should_epoch_stop": false,
-        "should_evaluate": false,
-        "should_log": false,
-        "should_save": true,
-        "should_training_stop": false
-      },
-      "attributes": {}
-    }
-  },
   "total_flos": 0.0,
   "train_batch_size": 8,
   "trial_name": null,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4cfcdc187e96bc5f72f21240b097691d6a8cdf8d0fcc5ad2fe20f43a43287471
-size 6520
+oid sha256:25686a2a94e778bd06e64992e121362fb8e396ce160a2ce4cb0004cc6efb88ff
+size 6264