Model save

Browse files

Files changed (6) hide show

README.md +78 -0
adapter_model.safetensors +1 -1
all_results.json +9 -0
runs/Jun05_04-41-00_7bdd13775218/events.out.tfevents.1717562483.7bdd13775218.60620.0 +2 -2
train_results.json +9 -0
trainer_state.json +299 -0

README.md ADDED Viewed

	@@ -0,0 +1,78 @@

+---
+license: gemma
+library_name: peft
+tags:
+- trl
+- sft
+- generated_from_trainer
+base_model: google/gemma-2b
+datasets:
+- generator
+model-index:
+- name: gemma2b-summarize-gemini1_5flash-8k
+  results: []
+---
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+# gemma2b-summarize-gemini1_5flash-8k
+This model is a fine-tuned version of [google/gemma-2b](https://huggingface.co/google/gemma-2b) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.5133
+## Model description
+More information needed
+## Intended uses & limitations
+More information needed
+## Training and evaluation data
+More information needed
+## Training procedure
+### Training hyperparameters
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 10
+### Training results
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 2.7544        | 0.9630 | 13   | 2.8722          |
+| 1.7723        | 2.0    | 27   | 2.6064          |
+| 1.4023        | 2.9630 | 40   | 2.5710          |
+| 1.2778        | 4.0    | 54   | 2.5349          |
+| 1.1848        | 4.9630 | 67   | 2.5176          |
+| 1.1522        | 6.0    | 81   | 2.5045          |
+| 1.1305        | 6.9630 | 94   | 2.5065          |
+| 1.1075        | 8.0    | 108  | 2.5136          |
+| 1.1049        | 8.9630 | 121  | 2.5129          |
+| 1.1048        | 9.6296 | 130  | 2.5133          |
+### Framework versions
+- PEFT 0.11.1
+- Transformers 4.40.1
+- Pytorch 2.2.0+cu121
+- Datasets 2.19.2
+- Tokenizers 0.19.1

adapter_model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:04403c70702e213b40b51a08ec413dc89fe64a46a4924056d80717e7806f25d9
 size 19644912

 version https://git-lfs.github.com/spec/v1
+oid sha256:cdcdbb5f08f94646f460cb0b194766f06ecf4dc3911c357818fe8b05fd0aa04e
 size 19644912

all_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "epoch": 9.62962962962963,
+    "total_flos": 1.018113810235392e+17,
+    "train_loss": 1.4375356710874116,
+    "train_runtime": 456.268,
+    "train_samples": 7919,
+    "train_samples_per_second": 18.301,
+    "train_steps_per_second": 0.285
+}

runs/Jun05_04-41-00_7bdd13775218/events.out.tfevents.1717562483.7bdd13775218.60620.0 CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eeaa39eb350c1555f5aaf958591bb51edcab04a3523b709678c27703b5d15f36
-size 11656

 version https://git-lfs.github.com/spec/v1
+oid sha256:6fe1d5ed141ea5a428a7f0149c31d790110c7191fa46a1f8319e55041803abe5
+size 14059

train_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "epoch": 9.62962962962963,
+    "total_flos": 1.018113810235392e+17,
+    "train_loss": 1.4375356710874116,
+    "train_runtime": 456.268,
+    "train_samples": 7919,
+    "train_samples_per_second": 18.301,
+    "train_steps_per_second": 0.285
+}

trainer_state.json ADDED Viewed

	@@ -0,0 +1,299 @@

+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 9.62962962962963,
+  "eval_steps": 500,
+  "global_step": 130,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.07407407407407407,
+      "grad_norm": 5.0625,
+      "learning_rate": 1.5384615384615387e-05,
+      "loss": 3.0102,
+      "step": 1
+    },
+    {
+      "epoch": 0.37037037037037035,
+      "grad_norm": 2.390625,
+      "learning_rate": 7.692307692307693e-05,
+      "loss": 3.0215,
+      "step": 5
+    },
+    {
+      "epoch": 0.7407407407407407,
+      "grad_norm": 11.5,
+      "learning_rate": 0.00015384615384615385,
+      "loss": 2.7544,
+      "step": 10
+    },
+    {
+      "epoch": 0.9629629629629629,
+      "eval_loss": 2.8721797466278076,
+      "eval_runtime": 0.652,
+      "eval_samples_per_second": 15.338,
+      "eval_steps_per_second": 1.534,
+      "step": 13
+    },
+    {
+      "epoch": 1.1111111111111112,
+      "grad_norm": 1.3671875,
+      "learning_rate": 0.00019985583705641418,
+      "loss": 2.379,
+      "step": 15
+    },
+    {
+      "epoch": 1.4814814814814814,
+      "grad_norm": 2.609375,
+      "learning_rate": 0.00019823877374156647,
+      "loss": 2.0489,
+      "step": 20
+    },
+    {
+      "epoch": 1.8518518518518519,
+      "grad_norm": 1.5234375,
+      "learning_rate": 0.00019485364419471454,
+      "loss": 1.7723,
+      "step": 25
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 2.60640549659729,
+      "eval_runtime": 0.5469,
+      "eval_samples_per_second": 18.283,
+      "eval_steps_per_second": 1.828,
+      "step": 27
+    },
+    {
+      "epoch": 2.2222222222222223,
+      "grad_norm": 0.66015625,
+      "learning_rate": 0.0001897613727639014,
+      "loss": 1.6022,
+      "step": 30
+    },
+    {
+      "epoch": 2.5925925925925926,
+      "grad_norm": 0.65234375,
+      "learning_rate": 0.00018305360832480117,
+      "loss": 1.4855,
+      "step": 35
+    },
+    {
+      "epoch": 2.962962962962963,
+      "grad_norm": 0.58203125,
+      "learning_rate": 0.00017485107481711012,
+      "loss": 1.4023,
+      "step": 40
+    },
+    {
+      "epoch": 2.962962962962963,
+      "eval_loss": 2.5709500312805176,
+      "eval_runtime": 0.7217,
+      "eval_samples_per_second": 13.857,
+      "eval_steps_per_second": 1.386,
+      "step": 40
+    },
+    {
+      "epoch": 3.3333333333333335,
+      "grad_norm": 0.458984375,
+      "learning_rate": 0.0001653013984983585,
+      "loss": 1.3253,
+      "step": 45
+    },
+    {
+      "epoch": 3.7037037037037037,
+      "grad_norm": 0.50390625,
+      "learning_rate": 0.00015457645101945046,
+      "loss": 1.2778,
+      "step": 50
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 2.5349316596984863,
+      "eval_runtime": 0.547,
+      "eval_samples_per_second": 18.283,
+      "eval_steps_per_second": 1.828,
+      "step": 54
+    },
+    {
+      "epoch": 4.074074074074074,
+      "grad_norm": 0.376953125,
+      "learning_rate": 0.00014286925614030542,
+      "loss": 1.2498,
+      "step": 55
+    },
+    {
+      "epoch": 4.444444444444445,
+      "grad_norm": 0.267578125,
+      "learning_rate": 0.0001303905157574247,
+      "loss": 1.2221,
+      "step": 60
+    },
+    {
+      "epoch": 4.814814814814815,
+      "grad_norm": 0.296875,
+      "learning_rate": 0.00011736481776669306,
+      "loss": 1.1848,
+      "step": 65
+    },
+    {
+      "epoch": 4.962962962962963,
+      "eval_loss": 2.5175788402557373,
+      "eval_runtime": 0.6693,
+      "eval_samples_per_second": 14.942,
+      "eval_steps_per_second": 1.494,
+      "step": 67
+    },
+    {
+      "epoch": 5.185185185185185,
+      "grad_norm": 0.291015625,
+      "learning_rate": 0.00010402659401094152,
+      "loss": 1.1814,
+      "step": 70
+    },
+    {
+      "epoch": 5.555555555555555,
+      "grad_norm": 0.33203125,
+      "learning_rate": 9.061590105968208e-05,
+      "loss": 1.1574,
+      "step": 75
+    },
+    {
+      "epoch": 5.925925925925926,
+      "grad_norm": 0.349609375,
+      "learning_rate": 7.73740997570278e-05,
+      "loss": 1.1522,
+      "step": 80
+    },
+    {
+      "epoch": 6.0,
+      "eval_loss": 2.5044538974761963,
+      "eval_runtime": 0.5444,
+      "eval_samples_per_second": 18.369,
+      "eval_steps_per_second": 1.837,
+      "step": 81
+    },
+    {
+      "epoch": 6.296296296296296,
+      "grad_norm": 0.2412109375,
+      "learning_rate": 6.453951129574644e-05,
+      "loss": 1.1367,
+      "step": 85
+    },
+    {
+      "epoch": 6.666666666666667,
+      "grad_norm": 0.28515625,
+      "learning_rate": 5.234312799786921e-05,
+      "loss": 1.1305,
+      "step": 90
+    },
+    {
+      "epoch": 6.962962962962963,
+      "eval_loss": 2.506514310836792,
+      "eval_runtime": 0.685,
+      "eval_samples_per_second": 14.598,
+      "eval_steps_per_second": 1.46,
+      "step": 94
+    },
+    {
+      "epoch": 7.037037037037037,
+      "grad_norm": 0.2265625,
+      "learning_rate": 4.100445599768774e-05,
+      "loss": 1.1188,
+      "step": 95
+    },
+    {
+      "epoch": 7.407407407407407,
+      "grad_norm": 0.2099609375,
+      "learning_rate": 3.072756464904006e-05,
+      "loss": 1.1222,
+      "step": 100
+    },
+    {
+      "epoch": 7.777777777777778,
+      "grad_norm": 0.248046875,
+      "learning_rate": 2.1697413758237784e-05,
+      "loss": 1.1075,
+      "step": 105
+    },
+    {
+      "epoch": 8.0,
+      "eval_loss": 2.5136494636535645,
+      "eval_runtime": 0.5462,
+      "eval_samples_per_second": 18.307,
+      "eval_steps_per_second": 1.831,
+      "step": 108
+    },
+    {
+      "epoch": 8.148148148148149,
+      "grad_norm": 0.2470703125,
+      "learning_rate": 1.4076524743778319e-05,
+      "loss": 1.1126,
+      "step": 110
+    },
+    {
+      "epoch": 8.518518518518519,
+      "grad_norm": 0.236328125,
+      "learning_rate": 8.002055634117578e-06,
+      "loss": 1.1118,
+      "step": 115
+    },
+    {
+      "epoch": 8.88888888888889,
+      "grad_norm": 0.2275390625,
+      "learning_rate": 3.5833325466437694e-06,
+      "loss": 1.1049,
+      "step": 120
+    },
+    {
+      "epoch": 8.962962962962964,
+      "eval_loss": 2.512882709503174,
+      "eval_runtime": 0.6584,
+      "eval_samples_per_second": 15.188,
+      "eval_steps_per_second": 1.519,
+      "step": 121
+    },
+    {
+      "epoch": 9.25925925925926,
+      "grad_norm": 0.2216796875,
+      "learning_rate": 8.998820754091531e-07,
+      "loss": 1.1116,
+      "step": 125
+    },
+    {
+      "epoch": 9.62962962962963,
+      "grad_norm": 0.20703125,
+      "learning_rate": 0.0,
+      "loss": 1.1048,
+      "step": 130
+    },
+    {
+      "epoch": 9.62962962962963,
+      "eval_loss": 2.513336658477783,
+      "eval_runtime": 0.5375,
+      "eval_samples_per_second": 18.606,
+      "eval_steps_per_second": 1.861,
+      "step": 130
+    },
+    {
+      "epoch": 9.62962962962963,
+      "step": 130,
+      "total_flos": 1.018113810235392e+17,
+      "train_loss": 1.4375356710874116,
+      "train_runtime": 456.268,
+      "train_samples_per_second": 18.301,
+      "train_steps_per_second": 0.285
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 130,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 100,
+  "total_flos": 1.018113810235392e+17,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}