Model save

README.md

@@ -13,7 +13,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model was trained from scratch on the None dataset.
+This model was trained from scratch on an unknown dataset.
 
 ## Model description
 
@@ -54,5 +54,5 @@ The following hyperparameters were used during training:
 
 - Transformers 4.39.3
 - Pytorch 2.1.2+cu118
-- Datasets 2.16.1
+- Datasets 2.19.1
 - Tokenizers 0.15.2

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6571147800988233,
-    "train_runtime": 7042.8127,
-    "train_samples": 61134,
-    "train_samples_per_second": 8.68,
-    "train_steps_per_second": 0.034
+    "train_loss": 1.2810401298381664,
+    "train_runtime": 5417.1403,
+    "train_samples": 48530,
+    "train_samples_per_second": 8.959,
+    "train_steps_per_second": 0.035
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/mnt/bn/xuruijie-llm/checkpoints/hh-rlhf/sft_0521/checkpoint-5500/",
+  "_name_or_path": "/mnt/bn/xuruijie-llm/checkpoints/new_world/v1-ultral",
   "architectures": [
     "MistralForCausalLM"
   ],

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dddc5c965ccc5628ea39a77853c3fb62741e797052df590decda94e2abd07848
+oid sha256:b008443197fefaac049fe7a0eabc01b0151b73564d92afc7767c1fa8c71e1ca8
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a54933b23f81e03fa674e855dbe4cff4d2b2db853148b612b2e9bda79732eef1
+oid sha256:9c3ff196fe6ec2e2cfd1db6b1aeb3c9df6fa0eb8b2d7025d4ac80bcd6c50ae14
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8eec202fc7af03b241019dcbe2b7d9864139f64498f3bed67407f5bca3a50ac2
+oid sha256:026ec669932b9619a7d776b17104d80d5e127fd258a8759a65b68cadf6158a75
 size 4540532728

runs/Jun14_02-10-31_n136-112-146/events.out.tfevents.1718302665.n136-112-146.3899146.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:98047bd374c280721085f729df31ccd128a61a18ee74e0c328ba033427961d5c
-size 12310
+oid sha256:4a5e7a536edc8e9aacc4b85472396013cb5053fa96f34d95849b2c2113aa398b
+size 18144

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6571147800988233,
-    "train_runtime": 7042.8127,
-    "train_samples": 61134,
-    "train_samples_per_second": 8.68,
-    "train_steps_per_second": 0.034
+    "train_loss": 1.2810401298381664,
+    "train_runtime": 5417.1403,
+    "train_samples": 48530,
+    "train_samples_per_second": 8.959,
+    "train_steps_per_second": 0.035
 }

trainer_state.json

@@ -1,22 +1,22 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 0.9960474308300395,
   "eval_steps": 500,
-  "global_step": 239,
+  "global_step": 189,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "grad_norm": 571.8916022631114,
-      "learning_rate": 2.083333333333333e-08,
-      "logits/chosen": -2.048940896987915,
-      "logits/rejected": -2.0532867908477783,
-      "logps/chosen": -384.4364318847656,
-      "logps/rejected": -422.8185729980469,
-      "loss": 1.3635,
+      "epoch": 0.01,
+      "grad_norm": 4737.779382861946,
+      "learning_rate": 2.6315789473684208e-08,
+      "logits/chosen": -4.638427734375,
+      "logits/rejected": -4.891327857971191,
+      "logps/chosen": -198.52749633789062,
+      "logps/rejected": -147.3392791748047,
+      "loss": 2.1269,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,362 +24,287 @@
       "step": 1
     },
     {
-      "epoch": 0.04,
-      "grad_norm": 402.57642784617445,
-      "learning_rate": 2.0833333333333333e-07,
-      "logits/chosen": -2.09702730178833,
-      "logits/rejected": -2.076167345046997,
-      "logps/chosen": -348.87298583984375,
-      "logps/rejected": -353.1369934082031,
-      "loss": 1.2239,
-      "rewards/accuracies": 0.4409722089767456,
-      "rewards/chosen": 0.08129607886075974,
-      "rewards/margins": 0.002124728402122855,
-      "rewards/rejected": 0.07917135208845139,
+      "epoch": 0.05,
+      "grad_norm": 3019.619079402316,
+      "learning_rate": 2.631578947368421e-07,
+      "logits/chosen": -4.496801376342773,
+      "logits/rejected": -4.816222190856934,
+      "logps/chosen": -224.27357482910156,
+      "logps/rejected": -168.04739379882812,
+      "loss": 1.9212,
+      "rewards/accuracies": 0.5381944179534912,
+      "rewards/chosen": 0.3541475236415863,
+      "rewards/margins": 0.37169286608695984,
+      "rewards/rejected": -0.017545383423566818,
       "step": 10
     },
     {
-      "epoch": 0.08,
-      "grad_norm": 222.2200617264089,
-      "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.1397833824157715,
-      "logits/rejected": -2.131530523300171,
-      "logps/chosen": -350.229736328125,
-      "logps/rejected": -320.4481506347656,
-      "loss": 0.941,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 1.970922827720642,
-      "rewards/margins": 0.47100192308425903,
-      "rewards/rejected": 1.4999210834503174,
+      "epoch": 0.11,
+      "grad_norm": 1863.4195630562826,
+      "learning_rate": 4.999573126145131e-07,
+      "logits/chosen": -4.533459663391113,
+      "logits/rejected": -4.848563194274902,
+      "logps/chosen": -220.4309539794922,
+      "logps/rejected": -180.72413635253906,
+      "loss": 1.1783,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": 5.777209281921387,
+      "rewards/margins": 6.331713676452637,
+      "rewards/rejected": -0.5545047521591187,
       "step": 20
     },
     {
-      "epoch": 0.13,
-      "grad_norm": 214.0593003301825,
-      "learning_rate": 4.990398100856366e-07,
-      "logits/chosen": -2.2208588123321533,
-      "logits/rejected": -2.1912357807159424,
-      "logps/chosen": -312.9959411621094,
-      "logps/rejected": -347.01055908203125,
-      "loss": 0.8736,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 4.563899040222168,
-      "rewards/margins": 0.5541414618492126,
-      "rewards/rejected": 4.009757041931152,
+      "epoch": 0.16,
+      "grad_norm": 1882.3555396757283,
+      "learning_rate": 4.948524419003415e-07,
+      "logits/chosen": -4.54370641708374,
+      "logits/rejected": -4.812285423278809,
+      "logps/chosen": -213.49411010742188,
+      "logps/rejected": -177.16848754882812,
+      "loss": 1.3518,
+      "rewards/accuracies": 0.8531249761581421,
+      "rewards/chosen": 8.2464017868042,
+      "rewards/margins": 12.621076583862305,
+      "rewards/rejected": -4.3746747970581055,
       "step": 30
     },
     {
-      "epoch": 0.17,
-      "grad_norm": 153.2749263573273,
-      "learning_rate": 4.931986719649298e-07,
-      "logits/chosen": -2.336960554122925,
-      "logits/rejected": -2.318669080734253,
-      "logps/chosen": -340.0647277832031,
-      "logps/rejected": -310.8158874511719,
-      "loss": 0.7797,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 4.682702541351318,
-      "rewards/margins": 1.4643404483795166,
-      "rewards/rejected": 3.218362331390381,
+      "epoch": 0.21,
+      "grad_norm": 1979.651061288252,
+      "learning_rate": 4.81409414945389e-07,
+      "logits/chosen": -4.619187831878662,
+      "logits/rejected": -4.8958845138549805,
+      "logps/chosen": -221.00082397460938,
+      "logps/rejected": -184.62203979492188,
+      "loss": 1.4689,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": 7.427975654602051,
+      "rewards/margins": 15.529205322265625,
+      "rewards/rejected": -8.101228713989258,
       "step": 40
     },
     {
-      "epoch": 0.21,
-      "grad_norm": 190.97179299432227,
-      "learning_rate": 4.821741763807186e-07,
-      "logits/chosen": -2.2911651134490967,
-      "logits/rejected": -2.2763919830322266,
-      "logps/chosen": -314.5186462402344,
-      "logps/rejected": -321.0877380371094,
-      "loss": 0.7046,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 2.596972942352295,
-      "rewards/margins": 1.727838158607483,
-      "rewards/rejected": 0.8691347241401672,
+      "epoch": 0.26,
+      "grad_norm": 2122.235483955093,
+      "learning_rate": 4.6008601790947314e-07,
+      "logits/chosen": -4.608691215515137,
+      "logits/rejected": -4.925226211547852,
+      "logps/chosen": -210.32058715820312,
+      "logps/rejected": -179.0367431640625,
+      "loss": 1.3821,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 6.428221225738525,
+      "rewards/margins": 16.42898941040039,
+      "rewards/rejected": -10.00076961517334,
       "step": 50
     },
     {
-      "epoch": 0.25,
-      "grad_norm": 164.07905629846462,
-      "learning_rate": 4.662012913161997e-07,
-      "logits/chosen": -2.275245428085327,
-      "logits/rejected": -2.253256320953369,
-      "logps/chosen": -356.04034423828125,
-      "logps/rejected": -334.3038635253906,
-      "loss": 0.6957,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": 3.08062481880188,
-      "rewards/margins": 1.801513671875,
-      "rewards/rejected": 1.2791111469268799,
+      "epoch": 0.32,
+      "grad_norm": 1692.2277360562514,
+      "learning_rate": 4.3160839350405605e-07,
+      "logits/chosen": -4.665585994720459,
+      "logits/rejected": -4.9272074699401855,
+      "logps/chosen": -205.7926788330078,
+      "logps/rejected": -178.56011962890625,
+      "loss": 1.3465,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 7.834652900695801,
+      "rewards/margins": 16.5399169921875,
+      "rewards/rejected": -8.705263137817383,
       "step": 60
     },
     {
-      "epoch": 0.29,
-      "grad_norm": 182.02883713121474,
-      "learning_rate": 4.456204510851956e-07,
-      "logits/chosen": -2.232480525970459,
-      "logits/rejected": -2.213390350341797,
-      "logps/chosen": -354.5909118652344,
-      "logps/rejected": -359.82049560546875,
-      "loss": 0.6379,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 2.7843456268310547,
-      "rewards/margins": 1.498595952987671,
-      "rewards/rejected": 1.2857494354248047,
+      "epoch": 0.37,
+      "grad_norm": 1810.8868167381333,
+      "learning_rate": 3.9694631307311825e-07,
+      "logits/chosen": -4.6464009284973145,
+      "logits/rejected": -4.913968086242676,
+      "logps/chosen": -207.1618194580078,
+      "logps/rejected": -182.61012268066406,
+      "loss": 1.3564,
+      "rewards/accuracies": 0.8656250238418579,
+      "rewards/chosen": 8.192334175109863,
+      "rewards/margins": 16.401655197143555,
+      "rewards/rejected": -8.209321975708008,
       "step": 70
     },
     {
-      "epoch": 0.33,
-      "grad_norm": 185.93895599972302,
-      "learning_rate": 4.2087030056579986e-07,
-      "logits/chosen": -2.2146525382995605,
-      "logits/rejected": -2.190126895904541,
-      "logps/chosen": -339.37750244140625,
-      "logps/rejected": -335.23565673828125,
-      "loss": 0.6529,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 2.3711397647857666,
-      "rewards/margins": 1.5012987852096558,
-      "rewards/rejected": 0.8698409199714661,
+      "epoch": 0.42,
+      "grad_norm": 1395.6821844604426,
+      "learning_rate": 3.572801521931522e-07,
+      "logits/chosen": -4.674800395965576,
+      "logits/rejected": -4.932587623596191,
+      "logps/chosen": -202.7789764404297,
+      "logps/rejected": -184.74395751953125,
+      "loss": 1.312,
+      "rewards/accuracies": 0.846875011920929,
+      "rewards/chosen": 5.3585076332092285,
+      "rewards/margins": 16.119762420654297,
+      "rewards/rejected": -10.761255264282227,
       "step": 80
     },
     {
-      "epoch": 0.38,
-      "grad_norm": 190.95476688008705,
-      "learning_rate": 3.9247834624635404e-07,
-      "logits/chosen": -2.1572606563568115,
-      "logits/rejected": -2.1407630443573,
-      "logps/chosen": -313.8312072753906,
-      "logps/rejected": -300.87176513671875,
-      "loss": 0.6248,
-      "rewards/accuracies": 0.653124988079071,
-      "rewards/chosen": 1.4995744228363037,
-      "rewards/margins": 1.1674902439117432,
-      "rewards/rejected": 0.3320842981338501,
+      "epoch": 0.47,
+      "grad_norm": 1761.719146022038,
+      "learning_rate": 3.139606943986089e-07,
+      "logits/chosen": -4.721759796142578,
+      "logits/rejected": -4.953747272491455,
+      "logps/chosen": -199.81448364257812,
+      "logps/rejected": -178.44004821777344,
+      "loss": 1.3425,
+      "rewards/accuracies": 0.815625011920929,
+      "rewards/chosen": 7.849789619445801,
+      "rewards/margins": 15.739909172058105,
+      "rewards/rejected": -7.890120029449463,
       "step": 90
     },
     {
-      "epoch": 0.42,
-      "grad_norm": 195.00852100658702,
-      "learning_rate": 3.610497133404795e-07,
-      "logits/chosen": -2.1591010093688965,
-      "logits/rejected": -2.1587300300598145,
-      "logps/chosen": -335.4873962402344,
-      "logps/rejected": -334.4920654296875,
-      "loss": 0.6092,
-      "rewards/accuracies": 0.671875,
-      "rewards/chosen": 1.2783663272857666,
-      "rewards/margins": 1.1912460327148438,
-      "rewards/rejected": 0.08712034672498703,
+      "epoch": 0.53,
+      "grad_norm": 1641.4466240114464,
+      "learning_rate": 2.684631318687185e-07,
+      "logits/chosen": -4.7313385009765625,
+      "logits/rejected": -4.984685897827148,
+      "logps/chosen": -213.2564239501953,
+      "logps/rejected": -190.69088745117188,
+      "loss": 1.3623,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 5.4120659828186035,
+      "rewards/margins": 16.620161056518555,
+      "rewards/rejected": -11.208093643188477,
       "step": 100
     },
     {
-      "epoch": 0.46,
-      "grad_norm": 192.15199269769496,
-      "learning_rate": 3.272542485937368e-07,
-      "logits/chosen": -2.1222145557403564,
-      "logits/rejected": -2.108656644821167,
-      "logps/chosen": -356.91278076171875,
-      "logps/rejected": -335.1573791503906,
-      "loss": 0.6316,
-      "rewards/accuracies": 0.671875,
-      "rewards/chosen": 0.5509830713272095,
-      "rewards/margins": 1.5539244413375854,
-      "rewards/rejected": -1.002941370010376,
+      "epoch": 0.58,
+      "grad_norm": 1346.9601711684072,
+      "learning_rate": 2.2233682952712483e-07,
+      "logits/chosen": -4.668034553527832,
+      "logits/rejected": -4.953825950622559,
+      "logps/chosen": -216.8499298095703,
+      "logps/rejected": -186.10470581054688,
+      "loss": 1.1234,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 7.529428005218506,
+      "rewards/margins": 15.865753173828125,
+      "rewards/rejected": -8.336324691772461,
       "step": 110
     },
     {
-      "epoch": 0.5,
-      "grad_norm": 199.26323132750235,
-      "learning_rate": 2.9181224366319943e-07,
-      "logits/chosen": -2.1514651775360107,
-      "logits/rejected": -2.13875150680542,
-      "logps/chosen": -335.4028015136719,
-      "logps/rejected": -338.54803466796875,
-      "loss": 0.5827,
-      "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": 0.3591001331806183,
-      "rewards/margins": 1.421007752418518,
-      "rewards/rejected": -1.0619075298309326,
+      "epoch": 0.63,
+      "grad_norm": 1999.3900490416042,
+      "learning_rate": 1.7715256327766884e-07,
+      "logits/chosen": -4.720789909362793,
+      "logits/rejected": -5.025943279266357,
+      "logps/chosen": -207.7978973388672,
+      "logps/rejected": -178.0445098876953,
+      "loss": 1.1185,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 8.364091873168945,
+      "rewards/margins": 16.011329650878906,
+      "rewards/rejected": -7.647237300872803,
       "step": 120
     },
     {
-      "epoch": 0.54,
-      "grad_norm": 183.666249275795,
-      "learning_rate": 2.55479083351317e-07,
-      "logits/chosen": -2.1675009727478027,
-      "logits/rejected": -2.150801420211792,
-      "logps/chosen": -380.80010986328125,
-      "logps/rejected": -352.38287353515625,
-      "loss": 0.5791,
-      "rewards/accuracies": 0.690625011920929,
-      "rewards/chosen": 0.7985715866088867,
-      "rewards/margins": 1.560523509979248,
-      "rewards/rejected": -0.7619519829750061,
+      "epoch": 0.69,
+      "grad_norm": 1568.9083661238265,
+      "learning_rate": 1.3444902911492174e-07,
+      "logits/chosen": -4.702408790588379,
+      "logits/rejected": -4.98063325881958,
+      "logps/chosen": -215.88174438476562,
+      "logps/rejected": -188.39645385742188,
+      "loss": 1.2748,
+      "rewards/accuracies": 0.8531249761581421,
+      "rewards/chosen": 7.107934474945068,
+      "rewards/margins": 16.361108779907227,
+      "rewards/rejected": -9.253173828125,
       "step": 130
     },
     {
-      "epoch": 0.59,
-      "grad_norm": 215.65073240301075,
-      "learning_rate": 2.19029145890313e-07,
-      "logits/chosen": -2.1623148918151855,
-      "logits/rejected": -2.1485044956207275,
-      "logps/chosen": -354.16107177734375,
-      "logps/rejected": -346.3798828125,
-      "loss": 0.5906,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.23650877177715302,
-      "rewards/margins": 1.3487697839736938,
-      "rewards/rejected": -1.1122608184814453,
+      "epoch": 0.74,
+      "grad_norm": 1390.666498149427,
+      "learning_rate": 9.56804446775518e-08,
+      "logits/chosen": -4.6129560470581055,
+      "logits/rejected": -4.923257350921631,
+      "logps/chosen": -208.4700469970703,
+      "logps/rejected": -178.78623962402344,
+      "loss": 1.0987,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": 6.485724449157715,
+      "rewards/margins": 17.193899154663086,
+      "rewards/rejected": -10.708174705505371,
       "step": 140
     },
     {
-      "epoch": 0.63,
-      "grad_norm": 187.86987212255403,
-      "learning_rate": 1.8323929841460178e-07,
-      "logits/chosen": -2.14518666267395,
-      "logits/rejected": -2.1396467685699463,
-      "logps/chosen": -357.54351806640625,
-      "logps/rejected": -344.2846984863281,
-      "loss": 0.5634,
-      "rewards/accuracies": 0.690625011920929,
-      "rewards/chosen": 0.020458679646253586,
-      "rewards/margins": 1.4657080173492432,
-      "rewards/rejected": -1.4452494382858276,
+      "epoch": 0.79,
+      "grad_norm": 1296.5821049110084,
+      "learning_rate": 6.216702761078166e-08,
+      "logits/chosen": -4.699868202209473,
+      "logits/rejected": -4.9864583015441895,
+      "logps/chosen": -196.650146484375,
+      "logps/rejected": -168.93551635742188,
+      "loss": 1.0856,
+      "rewards/accuracies": 0.8968750238418579,
+      "rewards/chosen": 7.758223056793213,
+      "rewards/margins": 17.15636444091797,
+      "rewards/rejected": -9.398139953613281,
       "step": 150
     },
     {
-      "epoch": 0.67,
-      "grad_norm": 201.55973529208768,
-      "learning_rate": 1.488723393865766e-07,
-      "logits/chosen": -2.127716541290283,
-      "logits/rejected": -2.107163667678833,
-      "logps/chosen": -385.68658447265625,
-      "logps/rejected": -347.5813903808594,
-      "loss": 0.5458,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.02848060429096222,
-      "rewards/margins": 1.421844244003296,
-      "rewards/rejected": -1.4503247737884521,
+      "epoch": 0.84,
+      "grad_norm": 1769.0071097352081,
+      "learning_rate": 3.5050037137906885e-08,
+      "logits/chosen": -4.634187698364258,
+      "logits/rejected": -4.958773612976074,
+      "logps/chosen": -211.03591918945312,
+      "logps/rejected": -176.72067260742188,
+      "loss": 1.1749,
+      "rewards/accuracies": 0.8531249761581421,
+      "rewards/chosen": 8.515033721923828,
+      "rewards/margins": 16.594724655151367,
+      "rewards/rejected": -8.079689979553223,
       "step": 160
     },
     {
-      "epoch": 0.71,
-      "grad_norm": 220.5429072103582,
-      "learning_rate": 1.1666074087171627e-07,
-      "logits/chosen": -2.1673145294189453,
-      "logits/rejected": -2.1441078186035156,
-      "logps/chosen": -364.5622253417969,
-      "logps/rejected": -367.15631103515625,
-      "loss": 0.5613,
-      "rewards/accuracies": 0.628125011920929,
-      "rewards/chosen": -0.00023489892191719264,
-      "rewards/margins": 1.2582697868347168,
-      "rewards/rejected": -1.2585046291351318,
+      "epoch": 0.9,
+      "grad_norm": 1606.699013433802,
+      "learning_rate": 1.5252909846235894e-08,
+      "logits/chosen": -4.62954044342041,
+      "logits/rejected": -4.913142204284668,
+      "logps/chosen": -209.8083953857422,
+      "logps/rejected": -184.52127075195312,
+      "loss": 1.2059,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 9.714839935302734,
+      "rewards/margins": 18.106443405151367,
+      "rewards/rejected": -8.391606330871582,
       "step": 170
     },
     {
-      "epoch": 0.75,
-      "grad_norm": 205.8312467056188,
-      "learning_rate": 8.729103716819111e-08,
-      "logits/chosen": -2.1819815635681152,
-      "logits/rejected": -2.148003101348877,
-      "logps/chosen": -365.36895751953125,
-      "logps/rejected": -343.6730041503906,
-      "loss": 0.5619,
-      "rewards/accuracies": 0.7281249761581421,
-      "rewards/chosen": 0.03583994507789612,
-      "rewards/margins": 1.5244814157485962,
-      "rewards/rejected": -1.4886412620544434,
+      "epoch": 0.95,
+      "grad_norm": 1242.5060745172418,
+      "learning_rate": 3.4498131616493565e-09,
+      "logits/chosen": -4.616083145141602,
+      "logits/rejected": -4.87780237197876,
+      "logps/chosen": -215.27685546875,
+      "logps/rejected": -193.55332946777344,
+      "loss": 1.1132,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 6.19677209854126,
+      "rewards/margins": 13.694157600402832,
+      "rewards/rejected": -7.497385501861572,
       "step": 180
     },
-    {
-      "epoch": 0.79,
-      "grad_norm": 248.5874406421173,
-      "learning_rate": 6.138919252022435e-08,
-      "logits/chosen": -2.1517252922058105,
-      "logits/rejected": -2.140963315963745,
-      "logps/chosen": -340.9563903808594,
-      "logps/rejected": -343.2937927246094,
-      "loss": 0.5592,
-      "rewards/accuracies": 0.640625,
-      "rewards/chosen": -0.48524799942970276,
-      "rewards/margins": 1.1504557132720947,
-      "rewards/rejected": -1.635703682899475,
-      "step": 190
-    },
-    {
-      "epoch": 0.84,
-      "grad_norm": 195.75027491525935,
-      "learning_rate": 3.9507259776993954e-08,
-      "logits/chosen": -2.1510136127471924,
-      "logits/rejected": -2.1407511234283447,
-      "logps/chosen": -375.94866943359375,
-      "logps/rejected": -375.7215576171875,
-      "loss": 0.5553,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.11154387891292572,
-      "rewards/margins": 1.316066861152649,
-      "rewards/rejected": -1.4276106357574463,
-      "step": 200
-    },
-    {
-      "epoch": 0.88,
-      "grad_norm": 193.59938752041137,
-      "learning_rate": 2.2111614344599684e-08,
-      "logits/chosen": -2.1561176776885986,
-      "logits/rejected": -2.128119468688965,
-      "logps/chosen": -384.52301025390625,
-      "logps/rejected": -371.85626220703125,
-      "loss": 0.5496,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.12478647381067276,
-      "rewards/margins": 1.6165813207626343,
-      "rewards/rejected": -1.4917947053909302,
-      "step": 210
-    },
-    {
-      "epoch": 0.92,
-      "grad_norm": 184.30698727614964,
-      "learning_rate": 9.57301420397924e-09,
-      "logits/chosen": -2.14475154876709,
-      "logits/rejected": -2.0955374240875244,
-      "logps/chosen": -371.14569091796875,
-      "logps/rejected": -362.03131103515625,
-      "loss": 0.5406,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.15589404106140137,
-      "rewards/margins": 1.2348332405090332,
-      "rewards/rejected": -1.3907272815704346,
-      "step": 220
-    },
-    {
-      "epoch": 0.96,
-      "grad_norm": 201.71570488034948,
-      "learning_rate": 2.158697848236607e-09,
-      "logits/chosen": -2.133373737335205,
-      "logits/rejected": -2.1216492652893066,
-      "logps/chosen": -373.65228271484375,
-      "logps/rejected": -375.40582275390625,
-      "loss": 0.5667,
-      "rewards/accuracies": 0.6781250238418579,
-      "rewards/chosen": -0.2453586757183075,
-      "rewards/margins": 1.2606414556503296,
-      "rewards/rejected": -1.5060001611709595,
-      "step": 230
-    },
     {
       "epoch": 1.0,
-      "step": 239,
+      "step": 189,
       "total_flos": 0.0,
-      "train_loss": 0.6571147800988233,
-      "train_runtime": 7042.8127,
-      "train_samples_per_second": 8.68,
-      "train_steps_per_second": 0.034
+      "train_loss": 1.2810401298381664,
+      "train_runtime": 5417.1403,
+      "train_samples_per_second": 8.959,
+      "train_steps_per_second": 0.035
     }
   ],
   "logging_steps": 10,
-  "max_steps": 239,
+  "max_steps": 189,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e370280be22422145d741eb20d38b32314505f586945952ee65047093ae07be1
+oid sha256:f7513499dfa8cc403e8936c05aa22586d81859c8852cb1c5f413ff049d51a71d
 size 6264