Model save

README.md

@@ -0,0 +1,78 @@
+---
+license: gemma
+library_name: peft
+tags:
+- trl
+- sft
+- generated_from_trainer
+base_model: google/gemma-2b
+datasets:
+- generator
+model-index:
+- name: gemma2b-summarize-gpt4o-8k
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# gemma2b-summarize-gpt4o-8k
+
+This model is a fine-tuned version of [google/gemma-2b](https://huggingface.co/google/gemma-2b) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.5343
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 3
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 48
+- total_eval_batch_size: 24
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 10
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 2.5077        | 0.9730 | 18   | 2.7787          |
+| 1.6701        | 2.0    | 37   | 2.6000          |
+| 1.3757        | 2.9730 | 55   | 2.5216          |
+| 1.2905        | 4.0    | 74   | 2.5137          |
+| 1.2291        | 4.9730 | 92   | 2.5113          |
+| 1.1946        | 6.0    | 111  | 2.5235          |
+| 1.1618        | 6.9730 | 129  | 2.5300          |
+| 1.1521        | 8.0    | 148  | 2.5335          |
+| 1.147         | 8.9730 | 166  | 2.5343          |
+| 1.14          | 9.7297 | 180  | 2.5343          |
+
+
+### Framework versions
+
+- PEFT 0.11.1
+- Transformers 4.41.2
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.2
+- Tokenizers 0.19.1

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4234580c34d453e1f8fe02895b5484387d715efd326eafb16b54eb80fc05a81
+oid sha256:4c414bfcb9dd16936c1c7e9a1c73d6cc3f19653be97a69f288cf9d83894a399a
 size 19644912

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 9.72972972972973,
+    "total_flos": 1.0572720102178816e+17,
+    "train_loss": 1.4424436648686727,
+    "train_runtime": 971.2544,
+    "train_samples": 8076,
+    "train_samples_per_second": 9.019,
+    "train_steps_per_second": 0.185
+}

runs/Jun05_13-55-05_user-HP-Z8-Fury-G5-Workstation-Desktop-PC/events.out.tfevents.1717563321.user-HP-Z8-Fury-G5-Workstation-Desktop-PC.25482.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:903f92038da3eb2160fee24289212eefcc729b0ba13db79c8af5f889da6ecab1
-size 11270
+oid sha256:4f6865a4510daa4ec2a3db5e020272a39576bb4277e879be750e87878e6a1362
+size 16330

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 9.72972972972973,
+    "total_flos": 1.0572720102178816e+17,
+    "train_loss": 1.4424436648686727,
+    "train_runtime": 971.2544,
+    "train_samples": 8076,
+    "train_samples_per_second": 9.019,
+    "train_steps_per_second": 0.185
+}

trainer_state.json

@@ -0,0 +1,381 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 9.72972972972973,
+  "eval_steps": 500,
+  "global_step": 180,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.05405405405405406,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.1111111111111112e-05,
+      "loss": 3.0425,
+      "step": 1
+    },
+    {
+      "epoch": 0.2702702702702703,
+      "grad_norm": 1.7890625,
+      "learning_rate": 5.555555555555556e-05,
+      "loss": 3.0524,
+      "step": 5
+    },
+    {
+      "epoch": 0.5405405405405406,
+      "grad_norm": 2.0625,
+      "learning_rate": 0.00011111111111111112,
+      "loss": 2.893,
+      "step": 10
+    },
+    {
+      "epoch": 0.8108108108108109,
+      "grad_norm": 1.359375,
+      "learning_rate": 0.0001666666666666667,
+      "loss": 2.5077,
+      "step": 15
+    },
+    {
+      "epoch": 0.972972972972973,
+      "eval_loss": 2.778700351715088,
+      "eval_runtime": 0.5506,
+      "eval_samples_per_second": 18.162,
+      "eval_steps_per_second": 1.816,
+      "step": 18
+    },
+    {
+      "epoch": 1.0810810810810811,
+      "grad_norm": 2.546875,
+      "learning_rate": 0.00019992479525042303,
+      "loss": 2.2381,
+      "step": 20
+    },
+    {
+      "epoch": 1.3513513513513513,
+      "grad_norm": 1.3671875,
+      "learning_rate": 0.00019908004033648453,
+      "loss": 1.9825,
+      "step": 25
+    },
+    {
+      "epoch": 1.6216216216216215,
+      "grad_norm": 82.0,
+      "learning_rate": 0.00019730448705798239,
+      "loss": 1.8071,
+      "step": 30
+    },
+    {
+      "epoch": 1.8918918918918919,
+      "grad_norm": 0.72265625,
+      "learning_rate": 0.00019461481568757506,
+      "loss": 1.6701,
+      "step": 35
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 2.600048065185547,
+      "eval_runtime": 0.5541,
+      "eval_samples_per_second": 18.046,
+      "eval_steps_per_second": 1.805,
+      "step": 37
+    },
+    {
+      "epoch": 2.1621621621621623,
+      "grad_norm": 0.5078125,
+      "learning_rate": 0.0001910362940966147,
+      "loss": 1.5495,
+      "step": 40
+    },
+    {
+      "epoch": 2.4324324324324325,
+      "grad_norm": 0.439453125,
+      "learning_rate": 0.00018660254037844388,
+      "loss": 1.4905,
+      "step": 45
+    },
+    {
+      "epoch": 2.7027027027027026,
+      "grad_norm": 0.376953125,
+      "learning_rate": 0.00018135520702629675,
+      "loss": 1.4373,
+      "step": 50
+    },
+    {
+      "epoch": 2.972972972972973,
+      "grad_norm": 0.25,
+      "learning_rate": 0.00017534358963276607,
+      "loss": 1.3757,
+      "step": 55
+    },
+    {
+      "epoch": 2.972972972972973,
+      "eval_loss": 2.521566152572632,
+      "eval_runtime": 0.5468,
+      "eval_samples_per_second": 18.289,
+      "eval_steps_per_second": 1.829,
+      "step": 55
+    },
+    {
+      "epoch": 3.2432432432432434,
+      "grad_norm": 0.474609375,
+      "learning_rate": 0.0001686241637868734,
+      "loss": 1.3452,
+      "step": 60
+    },
+    {
+      "epoch": 3.5135135135135136,
+      "grad_norm": 0.6171875,
+      "learning_rate": 0.0001612600545193203,
+      "loss": 1.315,
+      "step": 65
+    },
+    {
+      "epoch": 3.7837837837837838,
+      "grad_norm": 0.2734375,
+      "learning_rate": 0.00015332044328016914,
+      "loss": 1.2905,
+      "step": 70
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 2.51369309425354,
+      "eval_runtime": 0.5468,
+      "eval_samples_per_second": 18.287,
+      "eval_steps_per_second": 1.829,
+      "step": 74
+    },
+    {
+      "epoch": 4.054054054054054,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.00014487991802004623,
+      "loss": 1.2801,
+      "step": 75
+    },
+    {
+      "epoch": 4.324324324324325,
+      "grad_norm": 0.482421875,
+      "learning_rate": 0.00013601777248047105,
+      "loss": 1.2552,
+      "step": 80
+    },
+    {
+      "epoch": 4.594594594594595,
+      "grad_norm": 0.29296875,
+      "learning_rate": 0.00012681726127606376,
+      "loss": 1.2395,
+      "step": 85
+    },
+    {
+      "epoch": 4.864864864864865,
+      "grad_norm": 0.45703125,
+      "learning_rate": 0.00011736481776669306,
+      "loss": 1.2291,
+      "step": 90
+    },
+    {
+      "epoch": 4.972972972972973,
+      "eval_loss": 2.5113296508789062,
+      "eval_runtime": 0.5537,
+      "eval_samples_per_second": 18.061,
+      "eval_steps_per_second": 1.806,
+      "step": 92
+    },
+    {
+      "epoch": 5.135135135135135,
+      "grad_norm": 0.28515625,
+      "learning_rate": 0.0001077492420671931,
+      "loss": 1.2166,
+      "step": 95
+    },
+    {
+      "epoch": 5.405405405405405,
+      "grad_norm": 0.431640625,
+      "learning_rate": 9.806086682281758e-05,
+      "loss": 1.1963,
+      "step": 100
+    },
+    {
+      "epoch": 5.675675675675675,
+      "grad_norm": 0.408203125,
+      "learning_rate": 8.839070858747697e-05,
+      "loss": 1.2011,
+      "step": 105
+    },
+    {
+      "epoch": 5.945945945945946,
+      "grad_norm": 0.390625,
+      "learning_rate": 7.882961277705895e-05,
+      "loss": 1.1946,
+      "step": 110
+    },
+    {
+      "epoch": 6.0,
+      "eval_loss": 2.5234570503234863,
+      "eval_runtime": 0.5594,
+      "eval_samples_per_second": 17.877,
+      "eval_steps_per_second": 1.788,
+      "step": 111
+    },
+    {
+      "epoch": 6.216216216216216,
+      "grad_norm": 0.4765625,
+      "learning_rate": 6.94674002304887e-05,
+      "loss": 1.1733,
+      "step": 115
+    },
+    {
+      "epoch": 6.486486486486487,
+      "grad_norm": 0.3125,
+      "learning_rate": 6.039202339608432e-05,
+      "loss": 1.1803,
+      "step": 120
+    },
+    {
+      "epoch": 6.756756756756757,
+      "grad_norm": 0.326171875,
+      "learning_rate": 5.168874007033615e-05,
+      "loss": 1.1618,
+      "step": 125
+    },
+    {
+      "epoch": 6.972972972972973,
+      "eval_loss": 2.5299763679504395,
+      "eval_runtime": 0.5604,
+      "eval_samples_per_second": 17.843,
+      "eval_steps_per_second": 1.784,
+      "step": 129
+    },
+    {
+      "epoch": 7.027027027027027,
+      "grad_norm": 0.36328125,
+      "learning_rate": 4.343931245134616e-05,
+      "loss": 1.1584,
+      "step": 130
+    },
+    {
+      "epoch": 7.297297297297297,
+      "grad_norm": 0.357421875,
+      "learning_rate": 3.5721239031346066e-05,
+      "loss": 1.1484,
+      "step": 135
+    },
+    {
+      "epoch": 7.5675675675675675,
+      "grad_norm": 0.337890625,
+      "learning_rate": 2.8607026544210114e-05,
+      "loss": 1.1477,
+      "step": 140
+    },
+    {
+      "epoch": 7.837837837837838,
+      "grad_norm": 0.359375,
+      "learning_rate": 2.2163508807583998e-05,
+      "loss": 1.1521,
+      "step": 145
+    },
+    {
+      "epoch": 8.0,
+      "eval_loss": 2.5334644317626953,
+      "eval_runtime": 0.549,
+      "eval_samples_per_second": 18.214,
+      "eval_steps_per_second": 1.821,
+      "step": 148
+    },
+    {
+      "epoch": 8.108108108108109,
+      "grad_norm": 0.283203125,
+      "learning_rate": 1.6451218858706374e-05,
+      "loss": 1.1642,
+      "step": 150
+    },
+    {
+      "epoch": 8.378378378378379,
+      "grad_norm": 0.3046875,
+      "learning_rate": 1.1523820282334219e-05,
+      "loss": 1.1391,
+      "step": 155
+    },
+    {
+      "epoch": 8.64864864864865,
+      "grad_norm": 0.271484375,
+      "learning_rate": 7.427603073110967e-06,
+      "loss": 1.1491,
+      "step": 160
+    },
+    {
+      "epoch": 8.91891891891892,
+      "grad_norm": 0.2734375,
+      "learning_rate": 4.20104876845111e-06,
+      "loss": 1.147,
+      "step": 165
+    },
+    {
+      "epoch": 8.972972972972974,
+      "eval_loss": 2.534287929534912,
+      "eval_runtime": 0.5609,
+      "eval_samples_per_second": 17.829,
+      "eval_steps_per_second": 1.783,
+      "step": 166
+    },
+    {
+      "epoch": 9.18918918918919,
+      "grad_norm": 0.291015625,
+      "learning_rate": 1.874468937261531e-06,
+      "loss": 1.1527,
+      "step": 170
+    },
+    {
+      "epoch": 9.45945945945946,
+      "grad_norm": 0.283203125,
+      "learning_rate": 4.6972042068341714e-07,
+      "loss": 1.1486,
+      "step": 175
+    },
+    {
+      "epoch": 9.72972972972973,
+      "grad_norm": 0.28515625,
+      "learning_rate": 0.0,
+      "loss": 1.14,
+      "step": 180
+    },
+    {
+      "epoch": 9.72972972972973,
+      "eval_loss": 2.534336805343628,
+      "eval_runtime": 0.5494,
+      "eval_samples_per_second": 18.202,
+      "eval_steps_per_second": 1.82,
+      "step": 180
+    },
+    {
+      "epoch": 9.72972972972973,
+      "step": 180,
+      "total_flos": 1.0572720102178816e+17,
+      "train_loss": 1.4424436648686727,
+      "train_runtime": 971.2544,
+      "train_samples_per_second": 9.019,
+      "train_steps_per_second": 0.185
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 180,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.0572720102178816e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}