Model save

README.md

@@ -2,13 +2,11 @@
 license: other
 library_name: peft
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - sft
 - generated_from_trainer
 datasets:
-- zhihu
+- generator
 base_model: 01-ai/Yi-6B
 model-index:
 - name: Yi-6B-zhihu5
@@ -20,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # Yi-6B-zhihu5
 
-This model is a fine-tuned version of [01-ai/Yi-6B](https://huggingface.co/01-ai/Yi-6B) on the zhihu dataset.
+This model is a fine-tuned version of [01-ai/Yi-6B](https://huggingface.co/01-ai/Yi-6B) on the generator dataset.
 It achieves the following results on the evaluation set:
 - Loss: 2.5677
 
@@ -72,8 +70,8 @@ The following hyperparameters were used during training:
 | 2.5443        | 16.0  | 1536 | 2.5677          |
 | 2.5972        | 17.0  | 1632 | 2.5677          |
 | 2.5361        | 18.0  | 1728 | 2.5677          |
-| 2.5317        | 19.0  | 1824 | 2.5677          |
-| 2.632         | 20.0  | 1920 | 2.5677          |
+| 2.6119        | 19.0  | 1824 | 2.5677          |
+| 2.6321        | 20.0  | 1920 | 2.5677          |
 
 
 ### Framework versions

adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
     "o_proj",
-    "down_proj",
     "k_proj",
-    "v_proj",
+    "up_proj",
+    "gate_proj",
     "q_proj",
-    "up_proj"
+    "v_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bad84df15c90e0e8949fa9beda7034fd7e4534df74aba6ea1833c0495c075d13
+oid sha256:68d597f15641f407924e32d9666bbea85b9409352c7904467ce912e1804fb644
 size 72673912

all_results.json

@@ -1,13 +1,13 @@
 {
     "epoch": 20.0,
-    "eval_loss": 2.5677051544189453,
-    "eval_runtime": 164.7624,
+    "eval_loss": 2.5677125453948975,
+    "eval_runtime": 164.9005,
     "eval_samples": 2561,
-    "eval_samples_per_second": 4.649,
-    "eval_steps_per_second": 0.583,
-    "train_loss": 0.428011018037796,
-    "train_runtime": 2530.9924,
+    "eval_samples_per_second": 4.645,
+    "eval_steps_per_second": 0.582,
+    "train_loss": 0.16053936282793682,
+    "train_runtime": 1034.0653,
     "train_samples": 2561,
-    "train_samples_per_second": 6.053,
-    "train_steps_per_second": 0.759
+    "train_samples_per_second": 14.815,
+    "train_steps_per_second": 1.857
 }

eval_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 20.0,
-    "eval_loss": 2.5677051544189453,
-    "eval_runtime": 164.7624,
+    "eval_loss": 2.5677125453948975,
+    "eval_runtime": 164.9005,
     "eval_samples": 2561,
-    "eval_samples_per_second": 4.649,
-    "eval_steps_per_second": 0.583
+    "eval_samples_per_second": 4.645,
+    "eval_steps_per_second": 0.582
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 20.0,
-    "train_loss": 0.428011018037796,
-    "train_runtime": 2530.9924,
+    "train_loss": 0.16053936282793682,
+    "train_runtime": 1034.0653,
     "train_samples": 2561,
-    "train_samples_per_second": 6.053,
-    "train_steps_per_second": 0.759
+    "train_samples_per_second": 14.815,
+    "train_steps_per_second": 1.857
 }

trainer_state.json

@@ -2321,39 +2321,39 @@
     {
       "epoch": 18.8,
       "learning_rate": 1.0888432552681405e-07,
-      "loss": 2.6601,
+      "loss": 2.4623,
       "step": 1805
     },
     {
       "epoch": 18.85,
       "learning_rate": 9.965285027552452e-08,
-      "loss": 2.5226,
+      "loss": 2.538,
       "step": 1810
     },
     {
       "epoch": 18.91,
       "learning_rate": 9.082630146352356e-08,
-      "loss": 2.4359,
+      "loss": 2.5888,
       "step": 1815
     },
     {
       "epoch": 18.96,
       "learning_rate": 8.240540844791145e-08,
-      "loss": 2.5317,
+      "loss": 2.6119,
       "step": 1820
     },
     {
       "epoch": 19.0,
-      "eval_loss": 2.567706346511841,
-      "eval_runtime": 164.8443,
-      "eval_samples_per_second": 4.647,
+      "eval_loss": 2.567678689956665,
+      "eval_runtime": 164.8569,
+      "eval_samples_per_second": 4.646,
       "eval_steps_per_second": 0.582,
       "step": 1824
     },
     {
       "epoch": 19.01,
       "learning_rate": 7.439086706555743e-08,
-      "loss": 2.5861,
+      "loss": 2.6545,
       "step": 1825
     },
     {
@@ -2377,19 +2377,19 @@
     {
       "epoch": 19.22,
       "learning_rate": 4.640895825593683e-08,
-      "loss": 2.6011,
+      "loss": 2.6012,
       "step": 1845
     },
     {
       "epoch": 19.27,
       "learning_rate": 4.0435435515532304e-08,
-      "loss": 2.6357,
+      "loss": 2.6356,
       "step": 1850
     },
     {
       "epoch": 19.32,
       "learning_rate": 3.487173247935627e-08,
-      "loss": 2.5523,
+      "loss": 2.5524,
       "step": 1855
     },
     {
@@ -2401,19 +2401,19 @@
     {
       "epoch": 19.43,
       "learning_rate": 2.4975590581369778e-08,
-      "loss": 2.6604,
+      "loss": 2.6605,
       "step": 1865
     },
     {
       "epoch": 19.48,
       "learning_rate": 2.0643969459482326e-08,
-      "loss": 2.5515,
+      "loss": 2.5516,
       "step": 1870
     },
     {
       "epoch": 19.53,
       "learning_rate": 1.6723803454098408e-08,
-      "loss": 2.6021,
+      "loss": 2.6022,
       "step": 1875
     },
     {
@@ -2425,7 +2425,7 @@
     {
       "epoch": 19.64,
       "learning_rate": 1.0119098494316693e-08,
-      "loss": 2.4397,
+      "loss": 2.4395,
       "step": 1885
     },
     {
@@ -2437,7 +2437,7 @@
     {
       "epoch": 19.74,
       "learning_rate": 5.163658701989316e-09,
-      "loss": 2.4587,
+      "loss": 2.4588,
       "step": 1895
     },
     {
@@ -2455,7 +2455,7 @@
     {
       "epoch": 19.9,
       "learning_rate": 8.26304875812256e-10,
-      "loss": 2.6275,
+      "loss": 2.6277,
       "step": 1910
     },
     {
@@ -2467,13 +2467,13 @@
     {
       "epoch": 20.0,
       "learning_rate": 0.0,
-      "loss": 2.632,
+      "loss": 2.6321,
       "step": 1920
     },
     {
       "epoch": 20.0,
-      "eval_loss": 2.5677051544189453,
-      "eval_runtime": 164.798,
+      "eval_loss": 2.5677125453948975,
+      "eval_runtime": 164.791,
       "eval_samples_per_second": 4.648,
       "eval_steps_per_second": 0.583,
       "step": 1920
@@ -2482,10 +2482,10 @@
       "epoch": 20.0,
       "step": 1920,
       "total_flos": 1.0984887148766822e+18,
-      "train_loss": 0.428011018037796,
-      "train_runtime": 2530.9924,
-      "train_samples_per_second": 6.053,
-      "train_steps_per_second": 0.759
+      "train_loss": 0.16053936282793682,
+      "train_runtime": 1034.0653,
+      "train_samples_per_second": 14.815,
+      "train_steps_per_second": 1.857
     }
   ],
   "logging_steps": 5,