Model save

README.md

@@ -2,14 +2,9 @@
 license: mit
 base_model: ShenaoZhang/0.01_ablation_4iters_bs128_nodpo_iter_1
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - dpo
 - generated_from_trainer
-datasets:
-- updated
-- original
 model-index:
 - name: 0.01_ablation_4iters_bs128_nodpo_iter_2
   results: []
@@ -20,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # 0.01_ablation_4iters_bs128_nodpo_iter_2
 
-This model is a fine-tuned version of [ShenaoZhang/0.01_ablation_4iters_bs128_nodpo_iter_1](https://huggingface.co/ShenaoZhang/0.01_ablation_4iters_bs128_nodpo_iter_1) on the updated and the original datasets.
+This model is a fine-tuned version of [ShenaoZhang/0.01_ablation_4iters_bs128_nodpo_iter_1](https://huggingface.co/ShenaoZhang/0.01_ablation_4iters_bs128_nodpo_iter_1) on the None dataset.
 
 ## Model description
 

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.4913304132573745,
-    "train_runtime": 3553.949,
+    "train_loss": 0.5008462216673779,
+    "train_runtime": 3576.23,
     "train_samples": 15283,
-    "train_samples_per_second": 4.3,
+    "train_samples_per_second": 4.273,
     "train_steps_per_second": 0.033
 }

config.json

@@ -21,6 +21,6 @@
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.36.2",
-  "use_cache": true,
+  "use_cache": false,
   "vocab_size": 32000
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:22ffe23c047433d75550cf9b256a8d4eb0b30f0705b83e58b319acf1ec02f614
+oid sha256:636b4cedcb0312c208321733c21681d09266193b814b0aa53fe525721c240ad4
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be26fde64fb76d74226782cbe0f48a1d5b93d20b18ac17dbbfea045c152b8b7b
+oid sha256:f8cd6e69535273cb0000d1f28f84b8df3477963905ad75ffff50e5b78cf90100
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b5338a4ac27088d16b7f36c160c0e4b60498df26b230e74df08b44f706209a7
+oid sha256:7bdd2bc441aa26a2b7ff8505f6841c17bef188cadc87bbc2d8cf3bd020348caf
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.4913304132573745,
-    "train_runtime": 3553.949,
+    "train_loss": 0.5008462216673779,
+    "train_runtime": 3576.23,
     "train_samples": 15283,
-    "train_samples_per_second": 4.3,
+    "train_samples_per_second": 4.273,
     "train_steps_per_second": 0.033
 }

trainer_state.json

@@ -11,12 +11,12 @@
     {
       "epoch": 0.01,
       "learning_rate": 4.166666666666666e-08,
-      "logits/chosen": -2.613878011703491,
-      "logits/rejected": -2.5660829544067383,
-      "logps/chosen": -217.45541381835938,
-      "logps/pi_response": -177.27903747558594,
-      "logps/ref_response": -177.27903747558594,
-      "logps/rejected": -366.14935302734375,
+      "logits/chosen": -2.528585433959961,
+      "logits/rejected": -2.36845064163208,
+      "logps/chosen": -359.6035461425781,
+      "logps/pi_response": -267.75177001953125,
+      "logps/ref_response": -267.75177001953125,
+      "logps/rejected": -414.00299072265625,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -27,186 +27,186 @@
     {
       "epoch": 0.08,
       "learning_rate": 4.1666666666666667e-07,
-      "logits/chosen": -2.6187098026275635,
-      "logits/rejected": -2.5807690620422363,
-      "logps/chosen": -267.33782958984375,
-      "logps/pi_response": -188.70687866210938,
-      "logps/ref_response": -188.9645538330078,
-      "logps/rejected": -408.8757019042969,
-      "loss": 0.6806,
-      "rewards/accuracies": 0.5763888955116272,
-      "rewards/chosen": -0.011542385444045067,
-      "rewards/margins": 0.028939779847860336,
-      "rewards/rejected": -0.040482163429260254,
+      "logits/chosen": -2.517207145690918,
+      "logits/rejected": -2.463898181915283,
+      "logps/chosen": -328.0975341796875,
+      "logps/pi_response": -269.1342468261719,
+      "logps/ref_response": -269.0060119628906,
+      "logps/rejected": -410.38604736328125,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.4930555522441864,
+      "rewards/chosen": 0.0020175855606794357,
+      "rewards/margins": 0.008086067624390125,
+      "rewards/rejected": -0.006068482529371977,
       "step": 10
     },
     {
       "epoch": 0.17,
       "learning_rate": 4.931352528237397e-07,
-      "logits/chosen": -2.49855637550354,
-      "logits/rejected": -2.4376301765441895,
-      "logps/chosen": -310.2395324707031,
-      "logps/pi_response": -189.06040954589844,
-      "logps/ref_response": -197.4763641357422,
-      "logps/rejected": -463.3460388183594,
-      "loss": 0.6019,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.2946819067001343,
-      "rewards/margins": 0.44170132279396057,
-      "rewards/rejected": -0.7363831996917725,
+      "logits/chosen": -2.467337131500244,
+      "logits/rejected": -2.404489755630493,
+      "logps/chosen": -374.01763916015625,
+      "logps/pi_response": -322.4764709472656,
+      "logps/ref_response": -265.96600341796875,
+      "logps/rejected": -440.1626892089844,
+      "loss": 0.6083,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.3233157694339752,
+      "rewards/margins": 0.2321336269378662,
+      "rewards/rejected": -0.5554494261741638,
       "step": 20
     },
     {
       "epoch": 0.25,
       "learning_rate": 4.658920803689553e-07,
-      "logits/chosen": -2.4012527465820312,
-      "logits/rejected": -2.36314058303833,
-      "logps/chosen": -331.32171630859375,
-      "logps/pi_response": -186.35186767578125,
-      "logps/ref_response": -172.22605895996094,
-      "logps/rejected": -547.4236450195312,
-      "loss": 0.5577,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.7254283428192139,
-      "rewards/margins": 0.8743426203727722,
-      "rewards/rejected": -1.5997710227966309,
+      "logits/chosen": -2.250195264816284,
+      "logits/rejected": -2.1748290061950684,
+      "logps/chosen": -426.3290100097656,
+      "logps/pi_response": -466.5293884277344,
+      "logps/ref_response": -280.33502197265625,
+      "logps/rejected": -543.4297485351562,
+      "loss": 0.5736,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.9779343605041504,
+      "rewards/margins": 0.7406744360923767,
+      "rewards/rejected": -1.7186088562011719,
       "step": 30
     },
     {
       "epoch": 0.33,
       "learning_rate": 4.201712553872657e-07,
-      "logits/chosen": -2.3269619941711426,
-      "logits/rejected": -2.240555763244629,
-      "logps/chosen": -363.3132629394531,
-      "logps/pi_response": -221.7946014404297,
-      "logps/ref_response": -197.30540466308594,
-      "logps/rejected": -587.7777709960938,
-      "loss": 0.4939,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.7226591110229492,
-      "rewards/margins": 0.8834163546562195,
-      "rewards/rejected": -1.6060755252838135,
+      "logits/chosen": -2.1817965507507324,
+      "logits/rejected": -2.1026203632354736,
+      "logps/chosen": -485.78289794921875,
+      "logps/pi_response": -538.3165283203125,
+      "logps/ref_response": -283.85516357421875,
+      "logps/rejected": -658.8212280273438,
+      "loss": 0.5045,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.3960727453231812,
+      "rewards/margins": 1.1884379386901855,
+      "rewards/rejected": -2.584510564804077,
       "step": 40
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.598859066780754e-07,
-      "logits/chosen": -2.2625174522399902,
-      "logits/rejected": -2.212681293487549,
-      "logps/chosen": -370.35406494140625,
-      "logps/pi_response": -239.9530792236328,
-      "logps/ref_response": -191.80979919433594,
-      "logps/rejected": -613.0040893554688,
-      "loss": 0.4833,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -0.9313956499099731,
-      "rewards/margins": 1.1655590534210205,
-      "rewards/rejected": -2.096954822540283,
+      "logits/chosen": -2.1869587898254395,
+      "logits/rejected": -2.1076114177703857,
+      "logps/chosen": -507.6856994628906,
+      "logps/pi_response": -538.4204711914062,
+      "logps/ref_response": -288.7411193847656,
+      "logps/rejected": -600.2962036132812,
+      "loss": 0.4632,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -1.3790150880813599,
+      "rewards/margins": 0.8974047899246216,
+      "rewards/rejected": -2.2764198780059814,
       "step": 50
     },
     {
       "epoch": 0.5,
       "learning_rate": 2.9019570347986706e-07,
-      "logits/chosen": -2.123149871826172,
-      "logits/rejected": -2.0870766639709473,
-      "logps/chosen": -417.89910888671875,
-      "logps/pi_response": -260.7689514160156,
-      "logps/ref_response": -176.25942993164062,
-      "logps/rejected": -641.0624389648438,
-      "loss": 0.4766,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.3918354511260986,
-      "rewards/margins": 0.9974287152290344,
-      "rewards/rejected": -2.389263868331909,
+      "logits/chosen": -2.127645969390869,
+      "logits/rejected": -2.0537991523742676,
+      "logps/chosen": -503.4345703125,
+      "logps/pi_response": -531.116455078125,
+      "logps/ref_response": -279.95196533203125,
+      "logps/rejected": -640.44189453125,
+      "loss": 0.446,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.5597326755523682,
+      "rewards/margins": 1.0874087810516357,
+      "rewards/rejected": -2.647141218185425,
       "step": 60
     },
     {
       "epoch": 0.59,
       "learning_rate": 2.1706525253979534e-07,
-      "logits/chosen": -2.1073548793792725,
-      "logits/rejected": -2.020155429840088,
-      "logps/chosen": -420.85125732421875,
-      "logps/pi_response": -286.98138427734375,
-      "logps/ref_response": -186.5110626220703,
-      "logps/rejected": -682.7928466796875,
-      "loss": 0.4544,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.4277279376983643,
-      "rewards/margins": 1.2013485431671143,
-      "rewards/rejected": -2.6290764808654785,
+      "logits/chosen": -2.116987943649292,
+      "logits/rejected": -2.034440279006958,
+      "logps/chosen": -514.0672607421875,
+      "logps/pi_response": -500.27056884765625,
+      "logps/ref_response": -249.1424560546875,
+      "logps/rejected": -607.4042358398438,
+      "loss": 0.4319,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -1.739128828048706,
+      "rewards/margins": 0.8193766474723816,
+      "rewards/rejected": -2.5585055351257324,
       "step": 70
     },
     {
       "epoch": 0.67,
       "learning_rate": 1.4675360263490295e-07,
-      "logits/chosen": -2.016476631164551,
-      "logits/rejected": -1.9265648126602173,
-      "logps/chosen": -421.92877197265625,
-      "logps/pi_response": -293.125732421875,
-      "logps/ref_response": -171.429931640625,
-      "logps/rejected": -659.17822265625,
-      "loss": 0.4232,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.6673657894134521,
-      "rewards/margins": 1.0874935388565063,
-      "rewards/rejected": -2.754859447479248,
+      "logits/chosen": -2.014500379562378,
+      "logits/rejected": -1.956604242324829,
+      "logps/chosen": -505.2294921875,
+      "logps/pi_response": -537.6165771484375,
+      "logps/ref_response": -249.3338165283203,
+      "logps/rejected": -657.5457153320312,
+      "loss": 0.4453,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -1.9716640710830688,
+      "rewards/margins": 1.018070936203003,
+      "rewards/rejected": -2.9897356033325195,
       "step": 80
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.527854855097224e-08,
-      "logits/chosen": -2.043644428253174,
-      "logits/rejected": -1.8766616582870483,
-      "logps/chosen": -490.6349182128906,
-      "logps/pi_response": -333.29473876953125,
-      "logps/ref_response": -185.3270263671875,
-      "logps/rejected": -671.7935791015625,
-      "loss": 0.4292,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -2.052643299102783,
-      "rewards/margins": 0.9849675893783569,
-      "rewards/rejected": -3.0376105308532715,
+      "logits/chosen": -2.076690673828125,
+      "logits/rejected": -2.0011613368988037,
+      "logps/chosen": -553.7380981445312,
+      "logps/pi_response": -552.9459228515625,
+      "logps/ref_response": -259.7544860839844,
+      "logps/rejected": -676.1278686523438,
+      "loss": 0.455,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -2.102799892425537,
+      "rewards/margins": 1.045424222946167,
+      "rewards/rejected": -3.148223638534546,
       "step": 90
     },
     {
       "epoch": 0.84,
       "learning_rate": 3.790158337517127e-08,
-      "logits/chosen": -1.9523603916168213,
-      "logits/rejected": -1.9160430431365967,
-      "logps/chosen": -483.1397399902344,
-      "logps/pi_response": -349.1231994628906,
-      "logps/ref_response": -181.93067932128906,
-      "logps/rejected": -702.1567993164062,
-      "loss": 0.4485,
+      "logits/chosen": -2.011647939682007,
+      "logits/rejected": -1.9519059658050537,
+      "logps/chosen": -584.7652587890625,
+      "logps/pi_response": -556.218017578125,
+      "logps/ref_response": -270.5803527832031,
+      "logps/rejected": -668.7252197265625,
+      "loss": 0.4638,
       "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -2.182821273803711,
-      "rewards/margins": 0.9882111549377441,
-      "rewards/rejected": -3.171032428741455,
+      "rewards/chosen": -2.02268123626709,
+      "rewards/margins": 0.9910328984260559,
+      "rewards/rejected": -3.013713836669922,
       "step": 100
     },
     {
       "epoch": 0.92,
       "learning_rate": 8.677580722139671e-09,
-      "logits/chosen": -1.974956750869751,
-      "logits/rejected": -1.8683421611785889,
-      "logps/chosen": -464.55010986328125,
-      "logps/pi_response": -351.19903564453125,
-      "logps/ref_response": -178.7843780517578,
-      "logps/rejected": -691.6851196289062,
-      "loss": 0.4197,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -2.106511116027832,
-      "rewards/margins": 1.0789883136749268,
-      "rewards/rejected": -3.185499429702759,
+      "logits/chosen": -2.1074235439300537,
+      "logits/rejected": -1.9840141534805298,
+      "logps/chosen": -589.9749145507812,
+      "logps/pi_response": -545.8538818359375,
+      "logps/ref_response": -263.1878967285156,
+      "logps/rejected": -688.42529296875,
+      "loss": 0.4825,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -2.097093343734741,
+      "rewards/margins": 1.098931908607483,
+      "rewards/rejected": -3.1960251331329346,
       "step": 110
     },
     {
       "epoch": 1.0,
       "step": 119,
       "total_flos": 0.0,
-      "train_loss": 0.4913304132573745,
-      "train_runtime": 3553.949,
-      "train_samples_per_second": 4.3,
+      "train_loss": 0.5008462216673779,
+      "train_runtime": 3576.23,
+      "train_samples_per_second": 4.273,
       "train_steps_per_second": 0.033
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5fc9210e4528085e516084d985b3a15d6874f4e92aa31af54975e2c0cf8132d8
+oid sha256:8d422ab21f3020cc8f88fdb3a9a3c5233d2b1956f92df486f8a16b09196b42a6
 size 6008