Model save

README.md

@@ -2,13 +2,13 @@
 license: gemma
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - sft
+- alignment-handbook
 - generated_from_trainer
 base_model: google/gemma-2b
 datasets:
-- llama-duo/synth_summarize_dataset_dedup
+- generator
 model-index:
 - name: gemma2b-summarize-gpt4o-2k
   results: []
@@ -19,9 +19,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # gemma2b-summarize-gpt4o-2k
 
-This model is a fine-tuned version of [google/gemma-2b](https://huggingface.co/google/gemma-2b) on the llama-duo/synth_summarize_dataset_dedup dataset.
+This model is a fine-tuned version of [google/gemma-2b](https://huggingface.co/google/gemma-2b) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.5693
+- Loss: 2.5878
 
 ## Model description
 
@@ -58,16 +58,16 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 2.9969        | 1.0   | 5    | 3.1171          |
-| 2.5079        | 2.0   | 10   | 2.8491          |
-| 2.1709        | 3.0   | 15   | 2.7251          |
-| 1.9297        | 4.0   | 20   | 2.6267          |
-| 1.7591        | 5.0   | 25   | 2.5900          |
-| 1.6527        | 6.0   | 30   | 2.5908          |
-| 1.5938        | 7.0   | 35   | 2.5817          |
-| 1.5589        | 8.0   | 40   | 2.5729          |
-| 1.5434        | 9.0   | 45   | 2.5688          |
-| 1.5362        | 10.0  | 50   | 2.5693          |
+| 2.9978        | 1.0   | 5    | 3.1071          |
+| 2.5123        | 2.0   | 10   | 2.8503          |
+| 2.2077        | 3.0   | 15   | 2.7154          |
+| 1.9749        | 4.0   | 20   | 2.6507          |
+| 1.8015        | 5.0   | 25   | 2.6242          |
+| 1.6817        | 6.0   | 30   | 2.6105          |
+| 1.6095        | 7.0   | 35   | 2.6003          |
+| 1.5701        | 8.0   | 40   | 2.5917          |
+| 1.5524        | 9.0   | 45   | 2.5882          |
+| 1.5443        | 10.0  | 50   | 2.5878          |
 
 
 ### Framework versions

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
-    "k_proj",
+    "gate_proj",
     "v_proj",
-    "o_proj",
+    "q_proj",
+    "down_proj",
     "up_proj",
-    "gate_proj",
-    "down_proj"
+    "o_proj",
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:384152cc55936a07cedb1682ede49183b4d90927bdb646907aa3aefeabc34e92
+oid sha256:7b8e1cea41566ad451f2b5f26fee052cd2307ba926ad0df6f788db23e6de3911
 size 19644912

all_results.json

@@ -6,9 +6,9 @@
     "eval_samples_per_second": 18.91,
     "eval_steps_per_second": 1.891,
     "total_flos": 2.9368666998964224e+16,
-    "train_loss": 1.9250271797180176,
-    "train_runtime": 273.2217,
+    "train_loss": 1.9452767181396484,
+    "train_runtime": 274.5975,
     "train_samples": 2019,
-    "train_samples_per_second": 8.052,
-    "train_steps_per_second": 0.183
+    "train_samples_per_second": 8.012,
+    "train_steps_per_second": 0.182
 }

runs/Jun05_13-40-36_user-HP-Z8-Fury-G5-Workstation-Desktop-PC/events.out.tfevents.1717562450.user-HP-Z8-Fury-G5-Workstation-Desktop-PC.24815.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32214e1d82dead710cb009529df36d23bbd15a7098e0be493779095a0293a45b
+size 10878

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 10.0,
     "total_flos": 2.9368666998964224e+16,
-    "train_loss": 1.9250271797180176,
-    "train_runtime": 273.2217,
+    "train_loss": 1.9452767181396484,
+    "train_runtime": 274.5975,
     "train_samples": 2019,
-    "train_samples_per_second": 8.052,
-    "train_steps_per_second": 0.183
+    "train_samples_per_second": 8.012,
+    "train_steps_per_second": 0.182
 }

trainer_state.json

@@ -17,162 +17,162 @@
     },
     {
       "epoch": 1.0,
-      "grad_norm": 1.7265625,
+      "grad_norm": 1.65625,
       "learning_rate": 0.0002,
-      "loss": 2.9969,
+      "loss": 2.9978,
       "step": 5
     },
     {
       "epoch": 1.0,
-      "eval_loss": 3.1170578002929688,
-      "eval_runtime": 0.4884,
-      "eval_samples_per_second": 20.475,
-      "eval_steps_per_second": 2.047,
+      "eval_loss": 3.1071324348449707,
+      "eval_runtime": 0.5233,
+      "eval_samples_per_second": 19.11,
+      "eval_steps_per_second": 1.911,
       "step": 5
     },
     {
       "epoch": 2.0,
-      "grad_norm": 1.390625,
+      "grad_norm": 2.265625,
       "learning_rate": 0.00019396926207859084,
-      "loss": 2.5079,
+      "loss": 2.5123,
       "step": 10
     },
     {
       "epoch": 2.0,
-      "eval_loss": 2.849125862121582,
-      "eval_runtime": 0.5022,
-      "eval_samples_per_second": 19.912,
-      "eval_steps_per_second": 1.991,
+      "eval_loss": 2.8502731323242188,
+      "eval_runtime": 0.5332,
+      "eval_samples_per_second": 18.755,
+      "eval_steps_per_second": 1.876,
       "step": 10
     },
     {
       "epoch": 3.0,
-      "grad_norm": 1.2109375,
+      "grad_norm": 1.5,
       "learning_rate": 0.0001766044443118978,
-      "loss": 2.1709,
+      "loss": 2.2077,
       "step": 15
     },
     {
       "epoch": 3.0,
-      "eval_loss": 2.725147247314453,
-      "eval_runtime": 0.5172,
-      "eval_samples_per_second": 19.335,
-      "eval_steps_per_second": 1.934,
+      "eval_loss": 2.715447425842285,
+      "eval_runtime": 0.5429,
+      "eval_samples_per_second": 18.419,
+      "eval_steps_per_second": 1.842,
       "step": 15
     },
     {
       "epoch": 4.0,
-      "grad_norm": 0.8984375,
+      "grad_norm": 0.765625,
       "learning_rate": 0.00015000000000000001,
-      "loss": 1.9297,
+      "loss": 1.9749,
       "step": 20
     },
     {
       "epoch": 4.0,
-      "eval_loss": 2.6266896724700928,
-      "eval_runtime": 0.5348,
-      "eval_samples_per_second": 18.699,
-      "eval_steps_per_second": 1.87,
+      "eval_loss": 2.6506550312042236,
+      "eval_runtime": 0.54,
+      "eval_samples_per_second": 18.517,
+      "eval_steps_per_second": 1.852,
       "step": 20
     },
     {
       "epoch": 5.0,
-      "grad_norm": 0.796875,
+      "grad_norm": 0.7109375,
       "learning_rate": 0.00011736481776669306,
-      "loss": 1.7591,
+      "loss": 1.8015,
       "step": 25
     },
     {
       "epoch": 5.0,
-      "eval_loss": 2.5900256633758545,
-      "eval_runtime": 0.5346,
-      "eval_samples_per_second": 18.707,
-      "eval_steps_per_second": 1.871,
+      "eval_loss": 2.624201536178589,
+      "eval_runtime": 0.557,
+      "eval_samples_per_second": 17.953,
+      "eval_steps_per_second": 1.795,
       "step": 25
     },
     {
       "epoch": 6.0,
-      "grad_norm": 0.61328125,
+      "grad_norm": 0.369140625,
       "learning_rate": 8.263518223330697e-05,
-      "loss": 1.6527,
+      "loss": 1.6817,
       "step": 30
     },
     {
       "epoch": 6.0,
-      "eval_loss": 2.590770721435547,
-      "eval_runtime": 0.5433,
-      "eval_samples_per_second": 18.406,
-      "eval_steps_per_second": 1.841,
+      "eval_loss": 2.610503673553467,
+      "eval_runtime": 0.5392,
+      "eval_samples_per_second": 18.546,
+      "eval_steps_per_second": 1.855,
       "step": 30
     },
     {
       "epoch": 7.0,
-      "grad_norm": 0.341796875,
+      "grad_norm": 0.365234375,
       "learning_rate": 5.000000000000002e-05,
-      "loss": 1.5938,
+      "loss": 1.6095,
       "step": 35
     },
     {
       "epoch": 7.0,
-      "eval_loss": 2.5816967487335205,
-      "eval_runtime": 0.5475,
-      "eval_samples_per_second": 18.265,
-      "eval_steps_per_second": 1.826,
+      "eval_loss": 2.600292682647705,
+      "eval_runtime": 0.5472,
+      "eval_samples_per_second": 18.276,
+      "eval_steps_per_second": 1.828,
       "step": 35
     },
     {
       "epoch": 8.0,
-      "grad_norm": 0.322265625,
+      "grad_norm": 0.466796875,
       "learning_rate": 2.339555568810221e-05,
-      "loss": 1.5589,
+      "loss": 1.5701,
       "step": 40
     },
     {
       "epoch": 8.0,
-      "eval_loss": 2.5729336738586426,
-      "eval_runtime": 0.5448,
-      "eval_samples_per_second": 18.356,
-      "eval_steps_per_second": 1.836,
+      "eval_loss": 2.5916552543640137,
+      "eval_runtime": 0.5514,
+      "eval_samples_per_second": 18.135,
+      "eval_steps_per_second": 1.813,
       "step": 40
     },
     {
       "epoch": 9.0,
-      "grad_norm": 0.287109375,
+      "grad_norm": 0.3046875,
       "learning_rate": 6.030737921409169e-06,
-      "loss": 1.5434,
+      "loss": 1.5524,
       "step": 45
     },
     {
       "epoch": 9.0,
-      "eval_loss": 2.5688018798828125,
-      "eval_runtime": 0.5464,
-      "eval_samples_per_second": 18.302,
-      "eval_steps_per_second": 1.83,
+      "eval_loss": 2.5882315635681152,
+      "eval_runtime": 0.5445,
+      "eval_samples_per_second": 18.366,
+      "eval_steps_per_second": 1.837,
       "step": 45
     },
     {
       "epoch": 10.0,
-      "grad_norm": 0.310546875,
+      "grad_norm": 0.322265625,
       "learning_rate": 0.0,
-      "loss": 1.5362,
+      "loss": 1.5443,
       "step": 50
     },
     {
       "epoch": 10.0,
-      "eval_loss": 2.5693273544311523,
-      "eval_runtime": 0.5446,
-      "eval_samples_per_second": 18.363,
-      "eval_steps_per_second": 1.836,
+      "eval_loss": 2.587819814682007,
+      "eval_runtime": 0.5585,
+      "eval_samples_per_second": 17.905,
+      "eval_steps_per_second": 1.79,
       "step": 50
     },
     {
       "epoch": 10.0,
       "step": 50,
       "total_flos": 2.9368666998964224e+16,
-      "train_loss": 1.9250271797180176,
-      "train_runtime": 273.2217,
-      "train_samples_per_second": 8.052,
-      "train_steps_per_second": 0.183
+      "train_loss": 1.9452767181396484,
+      "train_runtime": 274.5975,
+      "train_samples_per_second": 8.012,
+      "train_steps_per_second": 0.182
     }
   ],
   "logging_steps": 5,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:538f26b6c9e5799ffada9b6a4eab53120a62cc6a5ea7b377dec56a152b4d1ee2
+oid sha256:d7baabb3fdff536acdfbe9b9b772fd100476be129edda2a86728e6b8b73db04c
 size 5304