Model save

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5566485422347293,
-    "train_runtime": 6841.2124,
+    "train_loss": 6.969558794025951,
+    "train_runtime": 6844.0218,
     "train_samples": 56236,
-    "train_samples_per_second": 8.22,
+    "train_samples_per_second": 8.217,
     "train_steps_per_second": 0.064
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.41.1",
+  "transformers_version": "4.39.3",
   "use_cache": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea0d09d1476d8a49fe285bd946375e1cbdcb6b821865850265935b691a3e4643
+oid sha256:e640ffc82529b75d69ac2fef13cf2d117f16eae6ce4736d73db1f1e86487b78a
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3cb1982972159f5dbb8063c84e14c867f92ca23eda59ef58fee5ea9fcd73336c
+oid sha256:297a6b580e9752eb5586014e574122b6431ca9c78e7aca1abc6337824516760d
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d179a041fb683849f3965d5a376a01e933a9c09cf0615f02ac2d59791e92b180
+oid sha256:fe3700b7100ceae0088757ea4c673b8e67e5727f459a539a1dd6471ce337ee14
 size 4540516344

runs/Jul02_16-21-36_n136-129-074/events.out.tfevents.1719908518.n136-129-074.2331461.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac24fcce5a4056e645486d804188b2a9cfa8d8adb16182fe4432538ebf41644d
-size 33009
+oid sha256:48799e24e4812fdb83269c2038d2ff01500a150f5083c1b5910eb65ae5df979e
+size 35427

tokenizer.json

@@ -134,7 +134,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5566485422347293,
-    "train_runtime": 6841.2124,
+    "train_loss": 6.969558794025951,
+    "train_runtime": 6844.0218,
     "train_samples": 56236,
-    "train_samples_per_second": 8.22,
+    "train_samples_per_second": 8.217,
     "train_steps_per_second": 0.064
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 113.90219393876704,
+      "grad_norm": 2276.7198889454276,
       "learning_rate": 2.2727272727272727e-09,
       "logits/chosen": -1.6768856048583984,
       "logits/rejected": -1.7259055376052856,
       "logps/chosen": -1.2793102264404297,
       "logps/rejected": -1.2162058353424072,
-      "loss": 0.6931,
+      "loss": 10.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,656 +25,656 @@
     },
     {
       "epoch": 0.02,
-      "grad_norm": 129.29173344209107,
+      "grad_norm": 2572.843914099358,
       "learning_rate": 2.2727272727272725e-08,
-      "logits/chosen": -1.702974796295166,
-      "logits/rejected": -1.6685913801193237,
-      "logps/chosen": -1.213309645652771,
-      "logps/rejected": -1.2203179597854614,
-      "loss": 0.6934,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -4.0181395888794214e-05,
-      "rewards/margins": -0.004325871355831623,
-      "rewards/rejected": 0.0042856899090111256,
+      "logits/chosen": -1.7036548852920532,
+      "logits/rejected": -1.668819785118103,
+      "logps/chosen": -1.2131812572479248,
+      "logps/rejected": -1.2204509973526,
+      "loss": 9.9973,
+      "rewards/accuracies": 0.4583333432674408,
+      "rewards/chosen": 0.01242614071816206,
+      "rewards/margins": -0.017121784389019012,
+      "rewards/rejected": 0.029547926038503647,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 129.013817774111,
+      "grad_norm": 2655.4615274085163,
       "learning_rate": 4.545454545454545e-08,
-      "logits/chosen": -1.7797605991363525,
-      "logits/rejected": -1.7345327138900757,
-      "logps/chosen": -1.144714593887329,
-      "logps/rejected": -1.1853687763214111,
-      "loss": 0.6906,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.014920851215720177,
-      "rewards/margins": 0.0041695646941661835,
-      "rewards/rejected": -0.01909041777253151,
+      "logits/chosen": -1.7792682647705078,
+      "logits/rejected": -1.7343353033065796,
+      "logps/chosen": -1.144930124282837,
+      "logps/rejected": -1.1850289106369019,
+      "loss": 9.9319,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -0.1707584410905838,
+      "rewards/margins": -0.013824567198753357,
+      "rewards/rejected": -0.15693387389183044,
       "step": 20
     },
     {
       "epoch": 0.07,
-      "grad_norm": 125.39732746183036,
+      "grad_norm": 2519.456913863188,
       "learning_rate": 6.818181818181817e-08,
-      "logits/chosen": -1.7443883419036865,
-      "logits/rejected": -1.6757313013076782,
-      "logps/chosen": -1.1939805746078491,
-      "logps/rejected": -1.2470417022705078,
-      "loss": 0.6755,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.11079101264476776,
-      "rewards/margins": 0.058313049376010895,
-      "rewards/rejected": -0.16910405457019806,
+      "logits/chosen": -1.7445284128189087,
+      "logits/rejected": -1.6756689548492432,
+      "logps/chosen": -1.195391297340393,
+      "logps/rejected": -1.2486639022827148,
+      "loss": 9.495,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -1.2489749193191528,
+      "rewards/margins": 0.6042808890342712,
+      "rewards/rejected": -1.8532556295394897,
       "step": 30
     },
     {
       "epoch": 0.09,
-      "grad_norm": 99.57108978782348,
+      "grad_norm": 1627.869026136585,
       "learning_rate": 9.09090909090909e-08,
-      "logits/chosen": -1.731239914894104,
-      "logits/rejected": -1.6651241779327393,
-      "logps/chosen": -1.2352855205535889,
-      "logps/rejected": -1.3083709478378296,
-      "loss": 0.6603,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.2833693027496338,
-      "rewards/margins": 0.20761942863464355,
-      "rewards/rejected": -0.49098867177963257,
+      "logits/chosen": -1.7305688858032227,
+      "logits/rejected": -1.664052963256836,
+      "logps/chosen": -1.247615098953247,
+      "logps/rejected": -1.3294904232025146,
+      "loss": 9.009,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.066654205322266,
+      "rewards/margins": 2.955174446105957,
+      "rewards/rejected": -7.021829128265381,
       "step": 40
     },
     {
       "epoch": 0.11,
-      "grad_norm": 94.95534987876165,
+      "grad_norm": 1748.050155291869,
       "learning_rate": 9.994307990108962e-08,
-      "logits/chosen": -1.6967604160308838,
-      "logits/rejected": -1.63302743434906,
-      "logps/chosen": -1.2550745010375977,
-      "logps/rejected": -1.304324984550476,
-      "loss": 0.6261,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.4698738157749176,
-      "rewards/margins": 0.29158082604408264,
-      "rewards/rejected": -0.7614546418190002,
+      "logits/chosen": -1.6962864398956299,
+      "logits/rejected": -1.6318918466567993,
+      "logps/chosen": -1.2720497846603394,
+      "logps/rejected": -1.3243694305419922,
+      "loss": 8.2497,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -6.396273612976074,
+      "rewards/margins": 3.222705125808716,
+      "rewards/rejected": -9.618978500366211,
       "step": 50
     },
     {
       "epoch": 0.14,
-      "grad_norm": 134.17306631598268,
+      "grad_norm": 2112.072704134999,
       "learning_rate": 9.959570405988094e-08,
-      "logits/chosen": -1.7350832223892212,
-      "logits/rejected": -1.6556373834609985,
-      "logps/chosen": -1.1744005680084229,
-      "logps/rejected": -1.2519078254699707,
-      "loss": 0.6292,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.6530232429504395,
-      "rewards/margins": 0.25578898191452026,
-      "rewards/rejected": -0.9088121652603149,
+      "logits/chosen": -1.7340772151947021,
+      "logits/rejected": -1.6545569896697998,
+      "logps/chosen": -1.1886461973190308,
+      "logps/rejected": -1.2660636901855469,
+      "loss": 8.3676,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -7.954795837402344,
+      "rewards/margins": 2.5489163398742676,
+      "rewards/rejected": -10.50371265411377,
       "step": 60
     },
     {
       "epoch": 0.16,
-      "grad_norm": 82.25189789646025,
+      "grad_norm": 1596.1626784723217,
       "learning_rate": 9.893476820924666e-08,
-      "logits/chosen": -1.8206403255462646,
-      "logits/rejected": -1.7381073236465454,
-      "logps/chosen": -1.2853193283081055,
-      "logps/rejected": -1.3663524389266968,
-      "loss": 0.5893,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -0.7247735261917114,
-      "rewards/margins": 0.34368234872817993,
-      "rewards/rejected": -1.0684559345245361,
+      "logits/chosen": -1.8238868713378906,
+      "logits/rejected": -1.740943193435669,
+      "logps/chosen": -1.2993048429489136,
+      "logps/rejected": -1.3821274042129517,
+      "loss": 7.5786,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -8.646288871765137,
+      "rewards/margins": 3.6157679557800293,
+      "rewards/rejected": -12.262056350708008,
       "step": 70
     },
     {
       "epoch": 0.18,
-      "grad_norm": 111.68081612756365,
+      "grad_norm": 1725.7540930385703,
       "learning_rate": 9.796445099843647e-08,
-      "logits/chosen": -1.8098180294036865,
-      "logits/rejected": -1.7271239757537842,
-      "logps/chosen": -1.2911165952682495,
-      "logps/rejected": -1.3880374431610107,
-      "loss": 0.6284,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.8414822816848755,
-      "rewards/margins": 0.3639078140258789,
-      "rewards/rejected": -1.2053899765014648,
+      "logits/chosen": -1.8160178661346436,
+      "logits/rejected": -1.7339365482330322,
+      "logps/chosen": -1.3012406826019287,
+      "logps/rejected": -1.4006680250167847,
+      "loss": 8.0893,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -9.427220344543457,
+      "rewards/margins": 3.889753818511963,
+      "rewards/rejected": -13.316973686218262,
       "step": 80
     },
     {
       "epoch": 0.2,
-      "grad_norm": 98.75254486198716,
+      "grad_norm": 1643.8851213987334,
       "learning_rate": 9.669088708527066e-08,
-      "logits/chosen": -1.7565476894378662,
-      "logits/rejected": -1.6922813653945923,
-      "logps/chosen": -1.3374165296554565,
-      "logps/rejected": -1.3871676921844482,
-      "loss": 0.5903,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.0171922445297241,
-      "rewards/margins": 0.42706432938575745,
-      "rewards/rejected": -1.4442565441131592,
+      "logits/chosen": -1.7639058828353882,
+      "logits/rejected": -1.7009646892547607,
+      "logps/chosen": -1.3530588150024414,
+      "logps/rejected": -1.4040849208831787,
+      "loss": 7.3762,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -11.736152648925781,
+      "rewards/margins": 4.398137092590332,
+      "rewards/rejected": -16.13429069519043,
       "step": 90
     },
     {
       "epoch": 0.23,
-      "grad_norm": 110.42472659036405,
+      "grad_norm": 1928.7061081884092,
       "learning_rate": 9.512212835085849e-08,
-      "logits/chosen": -1.7919187545776367,
-      "logits/rejected": -1.7078930139541626,
-      "logps/chosen": -1.284099817276001,
-      "logps/rejected": -1.3786065578460693,
-      "loss": 0.5835,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.1564443111419678,
-      "rewards/margins": 0.5179702639579773,
-      "rewards/rejected": -1.6744146347045898,
+      "logits/chosen": -1.799574613571167,
+      "logits/rejected": -1.715815782546997,
+      "logps/chosen": -1.2974745035171509,
+      "logps/rejected": -1.3930069208145142,
+      "loss": 7.3866,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -12.9019136428833,
+      "rewards/margins": 5.282271862030029,
+      "rewards/rejected": -18.184185028076172,
       "step": 100
     },
     {
       "epoch": 0.25,
-      "grad_norm": 94.2164442304763,
+      "grad_norm": 1572.512514314558,
       "learning_rate": 9.326809299301306e-08,
-      "logits/chosen": -1.7935024499893188,
-      "logits/rejected": -1.6928462982177734,
-      "logps/chosen": -1.3194783926010132,
-      "logps/rejected": -1.4348446130752563,
-      "loss": 0.5774,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.188363790512085,
-      "rewards/margins": 0.63763028383255,
-      "rewards/rejected": -1.8259938955307007,
+      "logits/chosen": -1.7981303930282593,
+      "logits/rejected": -1.6978185176849365,
+      "logps/chosen": -1.3314235210418701,
+      "logps/rejected": -1.4466179609298706,
+      "loss": 7.2883,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -13.078150749206543,
+      "rewards/margins": 6.359132289886475,
+      "rewards/rejected": -19.43728256225586,
       "step": 110
     },
     {
       "epoch": 0.27,
-      "grad_norm": 124.23898564185464,
+      "grad_norm": 1543.802209188709,
       "learning_rate": 9.114050282021158e-08,
-      "logits/chosen": -1.7868391275405884,
-      "logits/rejected": -1.7275841236114502,
-      "logps/chosen": -1.271259069442749,
-      "logps/rejected": -1.3735918998718262,
-      "loss": 0.563,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.2369645833969116,
-      "rewards/margins": 0.5494047403335571,
-      "rewards/rejected": -1.7863690853118896,
+      "logits/chosen": -1.7882074117660522,
+      "logits/rejected": -1.7293879985809326,
+      "logps/chosen": -1.286062479019165,
+      "logps/rejected": -1.391157865524292,
+      "loss": 7.1006,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -13.849987983703613,
+      "rewards/margins": 5.770316123962402,
+      "rewards/rejected": -19.62030601501465,
       "step": 120
     },
     {
       "epoch": 0.3,
-      "grad_norm": 94.58029537839892,
+      "grad_norm": 1776.0112703946616,
       "learning_rate": 8.875280914254802e-08,
-      "logits/chosen": -1.7857173681259155,
-      "logits/rejected": -1.6955511569976807,
-      "logps/chosen": -1.3196834325790405,
-      "logps/rejected": -1.4175035953521729,
-      "loss": 0.5654,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.2685333490371704,
-      "rewards/margins": 0.6939533948898315,
-      "rewards/rejected": -1.9624868631362915,
+      "logits/chosen": -1.7862787246704102,
+      "logits/rejected": -1.6958926916122437,
+      "logps/chosen": -1.3420274257659912,
+      "logps/rejected": -1.442373514175415,
+      "loss": 7.0019,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -14.9197416305542,
+      "rewards/margins": 7.192111015319824,
+      "rewards/rejected": -22.111852645874023,
       "step": 130
     },
     {
       "epoch": 0.32,
-      "grad_norm": 78.26072920533862,
+      "grad_norm": 1386.1262451501352,
       "learning_rate": 8.612010772821971e-08,
-      "logits/chosen": -1.8125206232070923,
-      "logits/rejected": -1.767704725265503,
-      "logps/chosen": -1.3238108158111572,
-      "logps/rejected": -1.362679123878479,
-      "loss": 0.5665,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.0993187427520752,
-      "rewards/margins": 0.6377816200256348,
-      "rewards/rejected": -1.73710036277771,
+      "logits/chosen": -1.8119525909423828,
+      "logits/rejected": -1.7669906616210938,
+      "logps/chosen": -1.3379199504852295,
+      "logps/rejected": -1.380608320236206,
+      "loss": 6.9963,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -12.404088020324707,
+      "rewards/margins": 6.759835243225098,
+      "rewards/rejected": -19.163921356201172,
       "step": 140
     },
     {
       "epoch": 0.34,
-      "grad_norm": 79.54962530262117,
+      "grad_norm": 1421.2461006997341,
       "learning_rate": 8.325904336322055e-08,
-      "logits/chosen": -1.784720778465271,
-      "logits/rejected": -1.728867530822754,
-      "logps/chosen": -1.2866287231445312,
-      "logps/rejected": -1.4028968811035156,
-      "loss": 0.565,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.192639708518982,
-      "rewards/margins": 0.6909436583518982,
-      "rewards/rejected": -1.883583426475525,
+      "logits/chosen": -1.784933090209961,
+      "logits/rejected": -1.7292178869247437,
+      "logps/chosen": -1.3000556230545044,
+      "logps/rejected": -1.41317617893219,
+      "loss": 7.1117,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -13.269081115722656,
+      "rewards/margins": 6.594681739807129,
+      "rewards/rejected": -19.8637638092041,
       "step": 150
     },
     {
       "epoch": 0.36,
-      "grad_norm": 81.2500983198601,
+      "grad_norm": 1409.35455155076,
       "learning_rate": 8.01877046176447e-08,
-      "logits/chosen": -1.7332093715667725,
-      "logits/rejected": -1.6599019765853882,
-      "logps/chosen": -1.3141100406646729,
-      "logps/rejected": -1.4209729433059692,
-      "loss": 0.5662,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.358100175857544,
-      "rewards/margins": 0.5547691583633423,
-      "rewards/rejected": -1.9128694534301758,
+      "logits/chosen": -1.733028769493103,
+      "logits/rejected": -1.659033179283142,
+      "logps/chosen": -1.3279526233673096,
+      "logps/rejected": -1.4333665370941162,
+      "loss": 7.0435,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -14.965250968933105,
+      "rewards/margins": 5.402812957763672,
+      "rewards/rejected": -20.368064880371094,
       "step": 160
     },
     {
       "epoch": 0.39,
-      "grad_norm": 82.5330575937106,
+      "grad_norm": 1409.9938243352115,
       "learning_rate": 7.692550948392249e-08,
-      "logits/chosen": -1.7886979579925537,
-      "logits/rejected": -1.7225795984268188,
-      "logps/chosen": -1.324988603591919,
-      "logps/rejected": -1.400198221206665,
-      "loss": 0.5593,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.1473289728164673,
-      "rewards/margins": 0.647649884223938,
-      "rewards/rejected": -1.7949787378311157,
+      "logits/chosen": -1.7891366481781006,
+      "logits/rejected": -1.7234008312225342,
+      "logps/chosen": -1.3368794918060303,
+      "logps/rejected": -1.410075068473816,
+      "loss": 7.0875,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -12.662399291992188,
+      "rewards/margins": 6.27509880065918,
+      "rewards/rejected": -18.937496185302734,
       "step": 170
     },
     {
       "epoch": 0.41,
-      "grad_norm": 90.45090476516394,
+      "grad_norm": 1576.9612574780165,
       "learning_rate": 7.349308261002021e-08,
-      "logits/chosen": -1.7442381381988525,
-      "logits/rejected": -1.6850671768188477,
-      "logps/chosen": -1.318302869796753,
-      "logps/rejected": -1.4245436191558838,
-      "loss": 0.5533,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.2121667861938477,
-      "rewards/margins": 0.5659047961235046,
-      "rewards/rejected": -1.778071641921997,
+      "logits/chosen": -1.7433929443359375,
+      "logits/rejected": -1.6840171813964844,
+      "logps/chosen": -1.3272547721862793,
+      "logps/rejected": -1.4322535991668701,
+      "loss": 6.8292,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -13.016866683959961,
+      "rewards/margins": 5.534828186035156,
+      "rewards/rejected": -18.551694869995117,
       "step": 180
     },
     {
       "epoch": 0.43,
-      "grad_norm": 84.71027965029923,
+      "grad_norm": 1468.7451103929668,
       "learning_rate": 6.991212490377531e-08,
-      "logits/chosen": -1.7995620965957642,
-      "logits/rejected": -1.745843529701233,
-      "logps/chosen": -1.3425184488296509,
-      "logps/rejected": -1.4334449768066406,
-      "loss": 0.5291,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.1241543292999268,
-      "rewards/margins": 0.6938456296920776,
-      "rewards/rejected": -1.818000078201294,
+      "logits/chosen": -1.7984473705291748,
+      "logits/rejected": -1.745666742324829,
+      "logps/chosen": -1.3522603511810303,
+      "logps/rejected": -1.4409388303756714,
+      "loss": 6.3646,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -12.215718269348145,
+      "rewards/margins": 6.713670253753662,
+      "rewards/rejected": -18.929386138916016,
       "step": 190
     },
     {
       "epoch": 0.46,
-      "grad_norm": 104.41708218825802,
+      "grad_norm": 1702.6463731612896,
       "learning_rate": 6.620527633276978e-08,
-      "logits/chosen": -1.7381435632705688,
-      "logits/rejected": -1.662602424621582,
-      "logps/chosen": -1.3073325157165527,
-      "logps/rejected": -1.4739927053451538,
-      "loss": 0.5404,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.1287935972213745,
-      "rewards/margins": 0.7987143993377686,
-      "rewards/rejected": -1.927507758140564,
+      "logits/chosen": -1.7361818552017212,
+      "logits/rejected": -1.6605857610702515,
+      "logps/chosen": -1.3193273544311523,
+      "logps/rejected": -1.4827220439910889,
+      "loss": 6.6324,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -12.487411499023438,
+      "rewards/margins": 7.660594940185547,
+      "rewards/rejected": -20.148006439208984,
       "step": 200
     },
     {
       "epoch": 0.48,
-      "grad_norm": 98.12458997212138,
+      "grad_norm": 1518.6969019476235,
       "learning_rate": 6.239597278716581e-08,
-      "logits/chosen": -1.808566689491272,
-      "logits/rejected": -1.7453174591064453,
-      "logps/chosen": -1.3598778247833252,
-      "logps/rejected": -1.4206109046936035,
-      "loss": 0.5355,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.1454766988754272,
-      "rewards/margins": 0.813967227935791,
-      "rewards/rejected": -1.9594440460205078,
+      "logits/chosen": -1.8051319122314453,
+      "logits/rejected": -1.7425997257232666,
+      "logps/chosen": -1.3682256937026978,
+      "logps/rejected": -1.4255290031433105,
+      "loss": 6.5008,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -12.289575576782227,
+      "rewards/margins": 7.796680450439453,
+      "rewards/rejected": -20.08625602722168,
       "step": 210
     },
     {
       "epoch": 0.5,
-      "grad_norm": 73.95327688888813,
+      "grad_norm": 1375.8319371273024,
       "learning_rate": 5.8508297910462456e-08,
-      "logits/chosen": -1.7581428289413452,
-      "logits/rejected": -1.6758205890655518,
-      "logps/chosen": -1.2785987854003906,
-      "logps/rejected": -1.4212281703948975,
-      "loss": 0.5267,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.1508338451385498,
-      "rewards/margins": 0.8485604524612427,
-      "rewards/rejected": -1.999394178390503,
+      "logits/chosen": -1.7532581090927124,
+      "logits/rejected": -1.6711580753326416,
+      "logps/chosen": -1.285611629486084,
+      "logps/rejected": -1.4231507778167725,
+      "loss": 6.3973,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -12.209630012512207,
+      "rewards/margins": 7.976588249206543,
+      "rewards/rejected": -20.186220169067383,
       "step": 220
     },
     {
       "epoch": 0.52,
-      "grad_norm": 101.92396024664481,
+      "grad_norm": 1612.3674111913235,
       "learning_rate": 5.456683083494731e-08,
-      "logits/chosen": -1.7508561611175537,
-      "logits/rejected": -1.7111085653305054,
-      "logps/chosen": -1.2999995946884155,
-      "logps/rejected": -1.3745485544204712,
-      "loss": 0.554,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.0619090795516968,
-      "rewards/margins": 0.5080693364143372,
-      "rewards/rejected": -1.5699783563613892,
+      "logits/chosen": -1.745671033859253,
+      "logits/rejected": -1.7057764530181885,
+      "logps/chosen": -1.3081130981445312,
+      "logps/rejected": -1.3810980319976807,
+      "loss": 6.8051,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -11.430437088012695,
+      "rewards/margins": 4.924307823181152,
+      "rewards/rejected": -16.3547420501709,
       "step": 230
     },
     {
       "epoch": 0.55,
-      "grad_norm": 110.90195766775575,
+      "grad_norm": 1819.5547604545825,
       "learning_rate": 5.059649078450834e-08,
-      "logits/chosen": -1.745862603187561,
-      "logits/rejected": -1.69782292842865,
-      "logps/chosen": -1.2861733436584473,
-      "logps/rejected": -1.4080326557159424,
-      "loss": 0.5271,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.1781983375549316,
-      "rewards/margins": 0.5982070565223694,
-      "rewards/rejected": -1.7764055728912354,
+      "logits/chosen": -1.7406162023544312,
+      "logits/rejected": -1.6928155422210693,
+      "logps/chosen": -1.2918732166290283,
+      "logps/rejected": -1.4103202819824219,
+      "loss": 6.3818,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -12.351980209350586,
+      "rewards/margins": 5.640812873840332,
+      "rewards/rejected": -17.992794036865234,
       "step": 240
     },
     {
       "epoch": 0.57,
-      "grad_norm": 90.78412484714156,
+      "grad_norm": 1555.9014610436766,
       "learning_rate": 4.6622379527277186e-08,
-      "logits/chosen": -1.7498047351837158,
-      "logits/rejected": -1.6966612339019775,
-      "logps/chosen": -1.263024091720581,
-      "logps/rejected": -1.3622833490371704,
-      "loss": 0.5322,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.241353988647461,
-      "rewards/margins": 0.6617964506149292,
-      "rewards/rejected": -1.9031503200531006,
+      "logits/chosen": -1.7455848455429077,
+      "logits/rejected": -1.6927179098129272,
+      "logps/chosen": -1.266522765159607,
+      "logps/rejected": -1.3599469661712646,
+      "loss": 6.4606,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -12.763422012329102,
+      "rewards/margins": 6.03444766998291,
+      "rewards/rejected": -18.797870635986328,
       "step": 250
     },
     {
       "epoch": 0.59,
-      "grad_norm": 80.47587413810997,
+      "grad_norm": 1440.8481572009775,
       "learning_rate": 4.26696226741691e-08,
-      "logits/chosen": -1.766274094581604,
-      "logits/rejected": -1.6926915645599365,
-      "logps/chosen": -1.3750711679458618,
-      "logps/rejected": -1.4678473472595215,
-      "loss": 0.5309,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -1.4224761724472046,
-      "rewards/margins": 0.7280942797660828,
-      "rewards/rejected": -2.1505703926086426,
+      "logits/chosen": -1.7621545791625977,
+      "logits/rejected": -1.6889396905899048,
+      "logps/chosen": -1.3751699924468994,
+      "logps/rejected": -1.4615355730056763,
+      "loss": 6.5489,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -14.234657287597656,
+      "rewards/margins": 6.639866828918457,
+      "rewards/rejected": -20.874523162841797,
       "step": 260
     },
     {
       "epoch": 0.61,
-      "grad_norm": 81.2536856430445,
+      "grad_norm": 1565.6866426892864,
       "learning_rate": 3.876321082668098e-08,
-      "logits/chosen": -1.8292741775512695,
-      "logits/rejected": -1.7543790340423584,
-      "logps/chosen": -1.3468070030212402,
-      "logps/rejected": -1.4356333017349243,
-      "loss": 0.5228,
+      "logits/chosen": -1.8229167461395264,
+      "logits/rejected": -1.7482731342315674,
+      "logps/chosen": -1.3493683338165283,
+      "logps/rejected": -1.4336580038070679,
+      "loss": 6.2912,
       "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.0831630229949951,
-      "rewards/margins": 0.7951737642288208,
-      "rewards/rejected": -1.8783369064331055,
+      "rewards/chosen": -11.087759017944336,
+      "rewards/margins": 7.498082637786865,
+      "rewards/rejected": -18.585840225219727,
       "step": 270
     },
     {
       "epoch": 0.64,
-      "grad_norm": 87.95396768106268,
+      "grad_norm": 1644.1893084856104,
       "learning_rate": 3.492784157826244e-08,
-      "logits/chosen": -1.754434585571289,
-      "logits/rejected": -1.654488205909729,
-      "logps/chosen": -1.331969141960144,
-      "logps/rejected": -1.4491215944290161,
-      "loss": 0.5257,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.0909301042556763,
-      "rewards/margins": 0.8176229596138,
-      "rewards/rejected": -1.908552885055542,
+      "logits/chosen": -1.7463124990463257,
+      "logits/rejected": -1.6467139720916748,
+      "logps/chosen": -1.3356127738952637,
+      "logps/rejected": -1.4482067823410034,
+      "loss": 6.3638,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -11.273643493652344,
+      "rewards/margins": 7.72042179107666,
+      "rewards/rejected": -18.994068145751953,
       "step": 280
     },
     {
       "epoch": 0.66,
-      "grad_norm": 75.75936548706386,
+      "grad_norm": 1246.6661658945,
       "learning_rate": 3.118776336817812e-08,
-      "logits/chosen": -1.800217628479004,
-      "logits/rejected": -1.7326066493988037,
-      "logps/chosen": -1.3206775188446045,
-      "logps/rejected": -1.4107261896133423,
-      "loss": 0.5236,
+      "logits/chosen": -1.7918853759765625,
+      "logits/rejected": -1.7234630584716797,
+      "logps/chosen": -1.3221752643585205,
+      "logps/rejected": -1.4063078165054321,
+      "loss": 6.2091,
       "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.1742886304855347,
-      "rewards/margins": 0.8283244967460632,
-      "rewards/rejected": -2.002613067626953,
+      "rewards/chosen": -11.892666816711426,
+      "rewards/margins": 7.691611289978027,
+      "rewards/rejected": -19.584278106689453,
       "step": 290
     },
     {
       "epoch": 0.68,
-      "grad_norm": 86.61923660491236,
+      "grad_norm": 1502.285401325102,
       "learning_rate": 2.7566622175067443e-08,
-      "logits/chosen": -1.781106948852539,
-      "logits/rejected": -1.7118265628814697,
-      "logps/chosen": -1.3446478843688965,
-      "logps/rejected": -1.4952938556671143,
-      "loss": 0.5304,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.3606449365615845,
-      "rewards/margins": 0.7265914082527161,
-      "rewards/rejected": -2.0872364044189453,
+      "logits/chosen": -1.7708361148834229,
+      "logits/rejected": -1.7010380029678345,
+      "logps/chosen": -1.349440336227417,
+      "logps/rejected": -1.4947826862335205,
+      "loss": 6.3971,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -14.085695266723633,
+      "rewards/margins": 6.735556125640869,
+      "rewards/rejected": -20.821250915527344,
       "step": 300
     },
     {
       "epoch": 0.71,
-      "grad_norm": 104.45230542444779,
+      "grad_norm": 2172.757810414492,
       "learning_rate": 2.408731201945432e-08,
-      "logits/chosen": -1.7766481637954712,
-      "logits/rejected": -1.7281579971313477,
-      "logps/chosen": -1.3146270513534546,
-      "logps/rejected": -1.4018762111663818,
-      "loss": 0.5176,
+      "logits/chosen": -1.7669150829315186,
+      "logits/rejected": -1.7179067134857178,
+      "logps/chosen": -1.3212790489196777,
+      "logps/rejected": -1.4049441814422607,
+      "loss": 6.286,
       "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.2073173522949219,
-      "rewards/margins": 0.670944333076477,
-      "rewards/rejected": -1.8782615661621094,
+      "rewards/chosen": -12.738364219665527,
+      "rewards/margins": 6.351044178009033,
+      "rewards/rejected": -19.08941078186035,
       "step": 310
     },
     {
       "epoch": 0.73,
-      "grad_norm": 97.89401056240122,
+      "grad_norm": 1676.044674890287,
       "learning_rate": 2.0771830220378112e-08,
-      "logits/chosen": -1.7322553396224976,
-      "logits/rejected": -1.6741504669189453,
-      "logps/chosen": -1.347646951675415,
-      "logps/rejected": -1.4063564538955688,
-      "loss": 0.5268,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2476258277893066,
-      "rewards/margins": 0.6655985713005066,
-      "rewards/rejected": -1.913224458694458,
+      "logits/chosen": -1.7216575145721436,
+      "logits/rejected": -1.6640005111694336,
+      "logps/chosen": -1.3518606424331665,
+      "logps/rejected": -1.4053070545196533,
+      "loss": 6.3543,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -12.897618293762207,
+      "rewards/margins": 6.129672050476074,
+      "rewards/rejected": -19.02729034423828,
       "step": 320
     },
     {
       "epoch": 0.75,
-      "grad_norm": 79.38110491324964,
+      "grad_norm": 1454.0668443582122,
       "learning_rate": 1.7641138321260257e-08,
-      "logits/chosen": -1.7678320407867432,
-      "logits/rejected": -1.692814826965332,
-      "logps/chosen": -1.3020561933517456,
-      "logps/rejected": -1.4196479320526123,
-      "loss": 0.5181,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -1.1820148229599,
-      "rewards/margins": 0.900565505027771,
-      "rewards/rejected": -2.082580327987671,
+      "logits/chosen": -1.7589213848114014,
+      "logits/rejected": -1.682925820350647,
+      "logps/chosen": -1.30251145362854,
+      "logps/rejected": -1.4143444299697876,
+      "loss": 6.1555,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -11.865683555603027,
+      "rewards/margins": 8.429769515991211,
+      "rewards/rejected": -20.295452117919922,
       "step": 330
     },
     {
       "epoch": 0.77,
-      "grad_norm": 79.242696260741,
+      "grad_norm": 1332.7467990696193,
       "learning_rate": 1.4715029564277793e-08,
-      "logits/chosen": -1.8388326168060303,
-      "logits/rejected": -1.7862517833709717,
-      "logps/chosen": -1.306144118309021,
-      "logps/rejected": -1.4227768182754517,
-      "loss": 0.5342,
+      "logits/chosen": -1.8296537399291992,
+      "logits/rejected": -1.776614785194397,
+      "logps/chosen": -1.3082842826843262,
+      "logps/rejected": -1.4200184345245361,
+      "loss": 6.5417,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.0148370265960693,
-      "rewards/margins": 0.7956900596618652,
-      "rewards/rejected": -1.8105270862579346,
+      "rewards/chosen": -10.362393379211426,
+      "rewards/margins": 7.467054843902588,
+      "rewards/rejected": -17.82944679260254,
       "step": 340
     },
     {
       "epoch": 0.8,
-      "grad_norm": 86.49780334887537,
+      "grad_norm": 1523.4091955188806,
       "learning_rate": 1.2012003751113343e-08,
-      "logits/chosen": -1.822198510169983,
-      "logits/rejected": -1.756326675415039,
-      "logps/chosen": -1.2951385974884033,
-      "logps/rejected": -1.4268760681152344,
-      "loss": 0.5045,
+      "logits/chosen": -1.8123096227645874,
+      "logits/rejected": -1.7457765340805054,
+      "logps/chosen": -1.2949508428573608,
+      "logps/rejected": -1.42155921459198,
+      "loss": 5.9983,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.329688310623169,
-      "rewards/margins": 0.815467357635498,
-      "rewards/rejected": -2.145155668258667,
+      "rewards/chosen": -13.278091430664062,
+      "rewards/margins": 7.641787528991699,
+      "rewards/rejected": -20.919878005981445,
       "step": 350
     },
     {
       "epoch": 0.82,
-      "grad_norm": 82.9744571713248,
+      "grad_norm": 1479.8993498955824,
       "learning_rate": 9.549150281252633e-09,
-      "logits/chosen": -1.7854712009429932,
-      "logits/rejected": -1.7307994365692139,
-      "logps/chosen": -1.2910486459732056,
-      "logps/rejected": -1.4215586185455322,
-      "loss": 0.508,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.1891343593597412,
-      "rewards/margins": 0.7877954244613647,
-      "rewards/rejected": -1.9769299030303955,
+      "logits/chosen": -1.7768176794052124,
+      "logits/rejected": -1.7211551666259766,
+      "logps/chosen": -1.2912156581878662,
+      "logps/rejected": -1.4147446155548096,
+      "loss": 6.1375,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -11.908048629760742,
+      "rewards/margins": 7.179844856262207,
+      "rewards/rejected": -19.0878963470459,
       "step": 360
     },
     {
       "epoch": 0.84,
-      "grad_norm": 77.94995731245406,
+      "grad_norm": 1484.9108288363668,
       "learning_rate": 7.3420401072985306e-09,
-      "logits/chosen": -1.8163235187530518,
-      "logits/rejected": -1.7626298666000366,
-      "logps/chosen": -1.3424322605133057,
-      "logps/rejected": -1.483293890953064,
-      "loss": 0.5133,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.2781879901885986,
-      "rewards/margins": 0.8339790105819702,
-      "rewards/rejected": -2.1121671199798584,
+      "logits/chosen": -1.8079217672348022,
+      "logits/rejected": -1.7539104223251343,
+      "logps/chosen": -1.3423258066177368,
+      "logps/rejected": -1.47573983669281,
+      "loss": 6.2086,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -12.771210670471191,
+      "rewards/margins": 7.595045566558838,
+      "rewards/rejected": -20.366256713867188,
       "step": 370
     },
     {
       "epoch": 0.86,
-      "grad_norm": 99.08582811249858,
+      "grad_norm": 1618.353367766913,
       "learning_rate": 5.404627290395369e-09,
-      "logits/chosen": -1.781881332397461,
-      "logits/rejected": -1.712882399559021,
-      "logps/chosen": -1.3003751039505005,
-      "logps/rejected": -1.4186222553253174,
-      "loss": 0.5136,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.1076927185058594,
-      "rewards/margins": 0.7825103998184204,
-      "rewards/rejected": -1.8902031183242798,
+      "logits/chosen": -1.7728559970855713,
+      "logits/rejected": -1.7030681371688843,
+      "logps/chosen": -1.3002992868423462,
+      "logps/rejected": -1.4108442068099976,
+      "loss": 6.298,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -11.069345474243164,
+      "rewards/margins": 7.0548906326293945,
+      "rewards/rejected": -18.124235153198242,
       "step": 380
     },
     {
       "epoch": 0.89,
-      "grad_norm": 142.606766968544,
+      "grad_norm": 2196.213456928754,
       "learning_rate": 3.74916077816162e-09,
-      "logits/chosen": -1.7914397716522217,
-      "logits/rejected": -1.7269777059555054,
-      "logps/chosen": -1.2829267978668213,
-      "logps/rejected": -1.3663567304611206,
-      "loss": 0.5283,
+      "logits/chosen": -1.7825977802276611,
+      "logits/rejected": -1.7169339656829834,
+      "logps/chosen": -1.2813310623168945,
+      "logps/rejected": -1.3594629764556885,
+      "loss": 6.404,
       "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.3313188552856445,
-      "rewards/margins": 0.6730116605758667,
-      "rewards/rejected": -2.0043303966522217,
+      "rewards/chosen": -13.153631210327148,
+      "rewards/margins": 6.200293064117432,
+      "rewards/rejected": -19.353923797607422,
       "step": 390
     },
     {
       "epoch": 0.91,
-      "grad_norm": 87.75942228209004,
+      "grad_norm": 1485.6207798922674,
       "learning_rate": 2.386106962899165e-09,
-      "logits/chosen": -1.7152767181396484,
-      "logits/rejected": -1.6405121088027954,
-      "logps/chosen": -1.3587360382080078,
-      "logps/rejected": -1.4870920181274414,
-      "loss": 0.523,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.3231509923934937,
-      "rewards/margins": 0.7201642394065857,
-      "rewards/rejected": -2.0433154106140137,
+      "logits/chosen": -1.7052252292633057,
+      "logits/rejected": -1.6294324398040771,
+      "logps/chosen": -1.357877492904663,
+      "logps/rejected": -1.4793249368667603,
+      "loss": 6.3409,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -13.145660400390625,
+      "rewards/margins": 6.510775089263916,
+      "rewards/rejected": -19.656435012817383,
       "step": 400
     },
     {
       "epoch": 0.93,
-      "grad_norm": 95.93932206444032,
+      "grad_norm": 1593.3048999278337,
       "learning_rate": 1.3240835096913706e-09,
-      "logits/chosen": -1.7500959634780884,
-      "logits/rejected": -1.6428474187850952,
-      "logps/chosen": -1.280632734298706,
-      "logps/rejected": -1.440298080444336,
-      "loss": 0.5334,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -1.0908193588256836,
-      "rewards/margins": 0.9025093913078308,
-      "rewards/rejected": -1.9933288097381592,
+      "logits/chosen": -1.7411197423934937,
+      "logits/rejected": -1.6328284740447998,
+      "logps/chosen": -1.2782161235809326,
+      "logps/rejected": -1.4311167001724243,
+      "loss": 6.5502,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -10.666521072387695,
+      "rewards/margins": 8.348638534545898,
+      "rewards/rejected": -19.015161514282227,
       "step": 410
     },
     {
       "epoch": 0.96,
-      "grad_norm": 185.3215699605587,
+      "grad_norm": 1557.0443174679092,
       "learning_rate": 5.698048727497462e-10,
-      "logits/chosen": -1.7732937335968018,
-      "logits/rejected": -1.6997044086456299,
-      "logps/chosen": -1.3186805248260498,
-      "logps/rejected": -1.473042607307434,
-      "loss": 0.5207,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -1.2738254070281982,
-      "rewards/margins": 0.8740528225898743,
-      "rewards/rejected": -2.1478781700134277,
+      "logits/chosen": -1.763695478439331,
+      "logits/rejected": -1.6889617443084717,
+      "logps/chosen": -1.3193089962005615,
+      "logps/rejected": -1.4645164012908936,
+      "loss": 6.3476,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -12.801119804382324,
+      "rewards/margins": 7.825028419494629,
+      "rewards/rejected": -20.626148223876953,
       "step": 420
     },
     {
       "epoch": 0.98,
-      "grad_norm": 85.90784633584286,
+      "grad_norm": 1384.5036857203058,
       "learning_rate": 1.2803984447259387e-10,
-      "logits/chosen": -1.7828824520111084,
-      "logits/rejected": -1.7222639322280884,
-      "logps/chosen": -1.310931921005249,
-      "logps/rejected": -1.4457513093948364,
-      "loss": 0.5116,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.2859268188476562,
-      "rewards/margins": 0.8716639280319214,
-      "rewards/rejected": -2.157590866088867,
+      "logits/chosen": -1.772894263267517,
+      "logits/rejected": -1.7122442722320557,
+      "logps/chosen": -1.3083751201629639,
+      "logps/rejected": -1.4367603063583374,
+      "loss": 6.1773,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -12.603564262390137,
+      "rewards/margins": 8.073251724243164,
+      "rewards/rejected": -20.67681884765625,
       "step": 430
     },
     {
       "epoch": 1.0,
       "step": 439,
       "total_flos": 0.0,
-      "train_loss": 0.5566485422347293,
-      "train_runtime": 6841.2124,
-      "train_samples_per_second": 8.22,
+      "train_loss": 6.969558794025951,
+      "train_runtime": 6844.0218,
+      "train_samples_per_second": 8.217,
       "train_steps_per_second": 0.064
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1987847d5ecd496557addb7e51b8b1d07a504e606fde07b6cd2c8c2d0088225b
-size 6520
+oid sha256:e588dab50a63217715e4ea48b897019c21921d9010f656a0ac29dbc3749cf6d4
+size 6328