update model checkpoint

README.md

@@ -1,14 +1,11 @@
 ---
-license: gemma
 library_name: peft
 tags:
 - alignment-handbook
-- trl
-- sft
 - generated_from_trainer
-base_model: google/gemma-7b
 datasets:
 - llama-duo/synth_summarize_dataset_dedup
+base_model: google/gemma-7b
 model-index:
 - name: gemma7b-summarize-gpt4o-1k
   results: []
@@ -21,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on the llama-duo/synth_summarize_dataset_dedup dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.8104
+- Loss: 8.6199
 
 ## Model description
 
@@ -42,13 +39,13 @@ More information needed
 The following hyperparameters were used during training:
 - learning_rate: 0.0002
 - train_batch_size: 4
-- eval_batch_size: 4
+- eval_batch_size: 2
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 2
+- num_devices: 8
 - gradient_accumulation_steps: 2
-- total_train_batch_size: 16
-- total_eval_batch_size: 8
+- total_train_batch_size: 64
+- total_eval_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
@@ -56,24 +53,24 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss |
-|:-------------:|:------:|:----:|:---------------:|
-| 37.3003       | 0.9231 | 6    | 12.3240         |
-| 21.0453       | 2.0    | 13   | 7.5884          |
-| 17.809        | 2.9231 | 19   | 5.6549          |
-| 3.5386        | 4.0    | 26   | 3.1607          |
-| 2.02          | 4.9231 | 32   | 2.8709          |
-| 1.732         | 6.0    | 39   | 2.7893          |
-| 1.4072        | 6.9231 | 45   | 2.7970          |
-| 1.3061        | 8.0    | 52   | 2.8125          |
-| 1.2613        | 8.9231 | 58   | 2.8098          |
-| 1.2304        | 9.2308 | 60   | 2.8104          |
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 45.5656       | 1.0   | 2    | 16.5046         |
+| 45.5656       | 2.0   | 4    | 14.2000         |
+| 35.6654       | 3.0   | 6    | 12.9944         |
+| 35.6654       | 4.0   | 8    | 11.5695         |
+| 22.2461       | 5.0   | 10   | 10.3065         |
+| 22.2461       | 6.0   | 12   | 9.3645          |
+| 22.2461       | 7.0   | 14   | 8.9071          |
+| 19.7508       | 8.0   | 16   | 8.6934          |
+| 19.7508       | 9.0   | 18   | 8.6287          |
+| 19.172        | 10.0  | 20   | 8.6199          |
 
 
 ### Framework versions
 
 - PEFT 0.10.0
 - Transformers 4.40.0
-- Pytorch 2.2.1+cu121
+- Pytorch 2.1.2+cu121
 - Datasets 2.18.0
 - Tokenizers 0.19.1

adapter_config.json

@@ -20,12 +20,7 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "up_proj",
-    "o_proj",
-    "down_proj",
     "q_proj",
-    "gate_proj",
     "v_proj"
   ],
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b104e6f5540b08ace63471b211aa65fe88708966056627cf08b1936b4b1d53a
-size 50056096
+oid sha256:8ba5026dc8a4332f1bda391e69642e0e08f75c80cded8fc13e67619b6649812b
+size 6437384

all_results.json

@@ -1,14 +1,14 @@
 {
-    "epoch": 9.23076923076923,
-    "eval_loss": 2.8104407787323,
-    "eval_runtime": 2.0654,
+    "epoch": 10.0,
+    "eval_loss": 8.61988639831543,
+    "eval_runtime": 0.2332,
     "eval_samples": 25,
-    "eval_samples_per_second": 4.842,
-    "eval_steps_per_second": 0.968,
-    "total_flos": 4.58661949299753e+16,
-    "train_loss": 8.682566889127095,
-    "train_runtime": 414.6045,
-    "train_samples": 923,
-    "train_samples_per_second": 2.484,
-    "train_steps_per_second": 0.145
+    "eval_samples_per_second": 42.89,
+    "eval_steps_per_second": 4.289,
+    "total_flos": 6.098355666236211e+16,
+    "train_loss": 24.70357437133789,
+    "train_runtime": 50.9898,
+    "train_samples": 1009,
+    "train_samples_per_second": 21.769,
+    "train_steps_per_second": 0.392
 }

config.json

@@ -23,9 +23,9 @@
     "_load_in_4bit": true,
     "_load_in_8bit": false,
     "bnb_4bit_compute_dtype": "bfloat16",
-    "bnb_4bit_quant_storage": "bfloat16",
+    "bnb_4bit_quant_storage": "uint8",
     "bnb_4bit_quant_type": "nf4",
-    "bnb_4bit_use_double_quant": true,
+    "bnb_4bit_use_double_quant": false,
     "llm_int8_enable_fp32_cpu_offload": false,
     "llm_int8_has_fp16_weight": false,
     "llm_int8_skip_modules": null,

eval_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 9.23076923076923,
-    "eval_loss": 2.8104407787323,
-    "eval_runtime": 2.0654,
+    "epoch": 10.0,
+    "eval_loss": 8.61988639831543,
+    "eval_runtime": 0.2332,
     "eval_samples": 25,
-    "eval_samples_per_second": 4.842,
-    "eval_steps_per_second": 0.968
+    "eval_samples_per_second": 42.89,
+    "eval_steps_per_second": 4.289
 }

runs/Jun13_05-28-24_gpu1-2/events.out.tfevents.1718227795.gpu1-2.1098203.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdd0a506e72160437f30230d4c3227102b55ed318f1b0059505260852041fe02
+size 9490

runs/Jun13_05-28-24_gpu1-2/events.out.tfevents.1718227846.gpu1-2.1098203.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed3a6b5fed727579c9ab25cc18830e87be1f35058e8396aea0dbb0a11ab82028
+size 354

runs/Jun13_05-43-12_gpu1-2/events.out.tfevents.1718228630.gpu1-2.1115325.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:325dfcd53bfd352d0daeed642f6bd91679bc5b6b0a03e3f41a7e5df262dcdd05
+size 9490

runs/Jun13_05-43-12_gpu1-2/events.out.tfevents.1718228682.gpu1-2.1115325.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8af5f66d4099cd4f2e4d3a9e694905bf6b6039f28e4bd1d81bcd158ddf908bf0
+size 354

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 9.23076923076923,
-    "total_flos": 4.58661949299753e+16,
-    "train_loss": 8.682566889127095,
-    "train_runtime": 414.6045,
-    "train_samples": 923,
-    "train_samples_per_second": 2.484,
-    "train_steps_per_second": 0.145
+    "epoch": 10.0,
+    "total_flos": 6.098355666236211e+16,
+    "train_loss": 24.70357437133789,
+    "train_runtime": 50.9898,
+    "train_samples": 1009,
+    "train_samples_per_second": 21.769,
+    "train_steps_per_second": 0.392
 }

trainer_state.json

@@ -1,200 +1,144 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 9.23076923076923,
+  "epoch": 10.0,
   "eval_steps": 500,
-  "global_step": 60,
+  "global_step": 20,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.15384615384615385,
-      "grad_norm": 390.0,
-      "learning_rate": 3.3333333333333335e-05,
-      "loss": 40.7928,
+      "epoch": 0.5,
+      "grad_norm": 173.0,
+      "learning_rate": 0.0001,
+      "loss": 45.5656,
       "step": 1
     },
     {
-      "epoch": 0.7692307692307693,
-      "grad_norm": 69.0,
-      "learning_rate": 0.0001666666666666667,
-      "loss": 37.3003,
-      "step": 5
-    },
-    {
-      "epoch": 0.9230769230769231,
-      "eval_loss": 12.323984146118164,
-      "eval_runtime": 2.0206,
-      "eval_samples_per_second": 4.949,
-      "eval_steps_per_second": 0.99,
-      "step": 6
-    },
-    {
-      "epoch": 1.5384615384615383,
-      "grad_norm": 26.0,
-      "learning_rate": 0.00019730448705798239,
-      "loss": 21.0453,
-      "step": 10
+      "epoch": 1.0,
+      "eval_loss": 16.504554748535156,
+      "eval_runtime": 0.2461,
+      "eval_samples_per_second": 40.635,
+      "eval_steps_per_second": 4.063,
+      "step": 2
     },
     {
       "epoch": 2.0,
-      "eval_loss": 7.588448524475098,
-      "eval_runtime": 2.0384,
-      "eval_samples_per_second": 4.906,
-      "eval_steps_per_second": 0.981,
-      "step": 13
+      "eval_loss": 14.199999809265137,
+      "eval_runtime": 0.2394,
+      "eval_samples_per_second": 41.776,
+      "eval_steps_per_second": 4.178,
+      "step": 4
     },
     {
-      "epoch": 2.3076923076923075,
-      "grad_norm": 16.75,
+      "epoch": 2.5,
+      "grad_norm": 31.5,
       "learning_rate": 0.00018660254037844388,
-      "loss": 17.809,
-      "step": 15
-    },
-    {
-      "epoch": 2.9230769230769234,
-      "eval_loss": 5.654947757720947,
-      "eval_runtime": 2.0499,
-      "eval_samples_per_second": 4.878,
-      "eval_steps_per_second": 0.976,
-      "step": 19
-    },
-    {
-      "epoch": 3.076923076923077,
-      "grad_norm": 58.0,
-      "learning_rate": 0.0001686241637868734,
-      "loss": 13.3008,
-      "step": 20
+      "loss": 35.6654,
+      "step": 5
     },
     {
-      "epoch": 3.8461538461538463,
-      "grad_norm": 8.0,
-      "learning_rate": 0.00014487991802004623,
-      "loss": 3.5386,
-      "step": 25
+      "epoch": 3.0,
+      "eval_loss": 12.994392395019531,
+      "eval_runtime": 0.2412,
+      "eval_samples_per_second": 41.452,
+      "eval_steps_per_second": 4.145,
+      "step": 6
     },
     {
       "epoch": 4.0,
-      "eval_loss": 3.160724401473999,
-      "eval_runtime": 2.0507,
-      "eval_samples_per_second": 4.876,
-      "eval_steps_per_second": 0.975,
-      "step": 26
+      "eval_loss": 11.569496154785156,
+      "eval_runtime": 0.2326,
+      "eval_samples_per_second": 42.992,
+      "eval_steps_per_second": 4.299,
+      "step": 8
     },
     {
-      "epoch": 4.615384615384615,
-      "grad_norm": 3.21875,
+      "epoch": 5.0,
+      "grad_norm": 10.5,
       "learning_rate": 0.00011736481776669306,
-      "loss": 2.02,
-      "step": 30
-    },
-    {
-      "epoch": 4.923076923076923,
-      "eval_loss": 2.8709278106689453,
-      "eval_runtime": 2.0507,
-      "eval_samples_per_second": 4.876,
-      "eval_steps_per_second": 0.975,
-      "step": 32
+      "loss": 22.2461,
+      "step": 10
     },
     {
-      "epoch": 5.384615384615385,
-      "grad_norm": 3.375,
-      "learning_rate": 8.839070858747697e-05,
-      "loss": 1.732,
-      "step": 35
+      "epoch": 5.0,
+      "eval_loss": 10.306487083435059,
+      "eval_runtime": 0.2322,
+      "eval_samples_per_second": 43.073,
+      "eval_steps_per_second": 4.307,
+      "step": 10
     },
     {
       "epoch": 6.0,
-      "eval_loss": 2.789313793182373,
-      "eval_runtime": 2.0444,
-      "eval_samples_per_second": 4.891,
-      "eval_steps_per_second": 0.978,
-      "step": 39
+      "eval_loss": 9.36452579498291,
+      "eval_runtime": 0.2332,
+      "eval_samples_per_second": 42.887,
+      "eval_steps_per_second": 4.289,
+      "step": 12
     },
     {
-      "epoch": 6.153846153846154,
-      "grad_norm": 4.0,
-      "learning_rate": 6.039202339608432e-05,
-      "loss": 1.5413,
-      "step": 40
+      "epoch": 7.0,
+      "eval_loss": 8.907111167907715,
+      "eval_runtime": 0.2345,
+      "eval_samples_per_second": 42.648,
+      "eval_steps_per_second": 4.265,
+      "step": 14
     },
     {
-      "epoch": 6.923076923076923,
-      "grad_norm": 2.015625,
+      "epoch": 7.5,
+      "grad_norm": 4.5,
       "learning_rate": 3.5721239031346066e-05,
-      "loss": 1.4072,
-      "step": 45
-    },
-    {
-      "epoch": 6.923076923076923,
-      "eval_loss": 2.797020435333252,
-      "eval_runtime": 2.049,
-      "eval_samples_per_second": 4.88,
-      "eval_steps_per_second": 0.976,
-      "step": 45
-    },
-    {
-      "epoch": 7.6923076923076925,
-      "grad_norm": 1.53125,
-      "learning_rate": 1.6451218858706374e-05,
-      "loss": 1.3061,
-      "step": 50
+      "loss": 19.7508,
+      "step": 15
     },
     {
       "epoch": 8.0,
-      "eval_loss": 2.8124804496765137,
-      "eval_runtime": 2.0441,
-      "eval_samples_per_second": 4.892,
-      "eval_steps_per_second": 0.978,
-      "step": 52
+      "eval_loss": 8.693410873413086,
+      "eval_runtime": 0.2425,
+      "eval_samples_per_second": 41.229,
+      "eval_steps_per_second": 4.123,
+      "step": 16
     },
     {
-      "epoch": 8.461538461538462,
-      "grad_norm": 1.453125,
-      "learning_rate": 4.20104876845111e-06,
-      "loss": 1.2613,
-      "step": 55
+      "epoch": 9.0,
+      "eval_loss": 8.628682136535645,
+      "eval_runtime": 0.2394,
+      "eval_samples_per_second": 41.765,
+      "eval_steps_per_second": 4.177,
+      "step": 18
     },
     {
-      "epoch": 8.923076923076923,
-      "eval_loss": 2.8097996711730957,
-      "eval_runtime": 2.0484,
-      "eval_samples_per_second": 4.882,
-      "eval_steps_per_second": 0.976,
-      "step": 58
-    },
-    {
-      "epoch": 9.23076923076923,
-      "grad_norm": 1.296875,
+      "epoch": 10.0,
+      "grad_norm": 4.125,
       "learning_rate": 0.0,
-      "loss": 1.2304,
-      "step": 60
+      "loss": 19.172,
+      "step": 20
     },
     {
-      "epoch": 9.23076923076923,
-      "eval_loss": 2.8104407787323,
-      "eval_runtime": 2.0425,
-      "eval_samples_per_second": 4.896,
-      "eval_steps_per_second": 0.979,
-      "step": 60
+      "epoch": 10.0,
+      "eval_loss": 8.61988639831543,
+      "eval_runtime": 0.232,
+      "eval_samples_per_second": 43.094,
+      "eval_steps_per_second": 4.309,
+      "step": 20
     },
     {
-      "epoch": 9.23076923076923,
-      "step": 60,
-      "total_flos": 4.58661949299753e+16,
-      "train_loss": 8.682566889127095,
-      "train_runtime": 414.6045,
-      "train_samples_per_second": 2.484,
-      "train_steps_per_second": 0.145
+      "epoch": 10.0,
+      "step": 20,
+      "total_flos": 6.098355666236211e+16,
+      "train_loss": 24.70357437133789,
+      "train_runtime": 50.9898,
+      "train_samples_per_second": 21.769,
+      "train_steps_per_second": 0.392
     }
   ],
   "logging_steps": 5,
-  "max_steps": 60,
+  "max_steps": 20,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 10,
   "save_steps": 100,
-  "total_flos": 4.58661949299753e+16,
+  "total_flos": 6.098355666236211e+16,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d621e364c8574aed19ac641cf9318bb861f0211c8e6432fdb14829c73625eadd
+oid sha256:e21acb43b827015a9b74447a5fa950df5fd32ebdb581d53558420047a768950e
 size 5176