Model save

README.md

@@ -2,15 +2,11 @@
 license: apache-2.0
 base_model: JackFram/llama-68m
 tags:
-- alignment-handbook
-- trl
-- sft
-- generated_from_trainer
 - trl
 - sft
 - generated_from_trainer
 datasets:
-- GAIR/lima
+- generator
 model-index:
 - name: gpt2-sft-lima
   results: []
@@ -21,9 +17,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # gpt2-sft-lima
 
-This model is a fine-tuned version of [JackFram/llama-68m](https://huggingface.co/JackFram/llama-68m) on the GAIR/lima dataset.
+This model is a fine-tuned version of [JackFram/llama-68m](https://huggingface.co/JackFram/llama-68m) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 5.4846
+- Loss: 5.5083
 
 ## Model description
 
@@ -46,6 +42,10 @@ The following hyperparameters were used during training:
 - train_batch_size: 32
 - eval_batch_size: 8
 - seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 128
+- total_eval_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
@@ -55,13 +55,13 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 3.4931        | 1.0   | 24   | 5.5962          |
-| 3.4278        | 2.0   | 48   | 5.4846          |
+| 3.6492        | 1.0   | 6    | 5.7124          |
+| 3.457         | 2.0   | 12   | 5.5083          |
 
 
 ### Framework versions
 
-- Transformers 4.40.1
+- Transformers 4.38.2
 - Pytorch 2.1.2
 - Datasets 2.14.6
-- Tokenizers 0.19.1
+- Tokenizers 0.15.2

all_results.json

@@ -1,14 +1,8 @@
 {
     "epoch": 2.0,
-    "eval_loss": 5.484647274017334,
-    "eval_runtime": 0.045,
-    "eval_samples": 300,
-    "eval_samples_per_second": 266.847,
-    "eval_steps_per_second": 44.475,
-    "total_flos": 394600362541056.0,
-    "train_loss": 3.4869212259848914,
-    "train_runtime": 14.4504,
+    "train_loss": 3.5401106079419455,
+    "train_runtime": 10.4273,
     "train_samples": 1030,
-    "train_samples_per_second": 102.281,
-    "train_steps_per_second": 3.322
+    "train_samples_per_second": 141.744,
+    "train_steps_per_second": 1.151
 }

config.json

@@ -23,7 +23,7 @@
   "rope_theta": 10000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.1",
-  "use_cache": true,
+  "transformers_version": "4.38.2",
+  "use_cache": false,
   "vocab_size": 32000
 }

generation_config.json

@@ -3,5 +3,5 @@
   "bos_token_id": 0,
   "eos_token_id": 2,
   "pad_token_id": 1,
-  "transformers_version": "4.40.1"
+  "transformers_version": "4.38.2"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d8806bf32cff7ec1911f05503bac5b2c0db8784dc06f7b1ec97ab24b7c4851c2
+oid sha256:08f0b8f3e809f4bed4446ca5ad5539a8088deadbf434fda96d9dee7d88c1749f
 size 136062744

runs/Apr25_15-24-23_aga39/events.out.tfevents.1714076671.aga39.828093.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df9530b3fa9dddfd0c29be52b1d8862925277dea587a847edc911e8c1f0a6b57
+size 6203

tokenizer.json

@@ -134,7 +134,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,9 +1,8 @@
 {
     "epoch": 2.0,
-    "total_flos": 394600362541056.0,
-    "train_loss": 3.4869212259848914,
-    "train_runtime": 14.4504,
+    "train_loss": 3.5401106079419455,
+    "train_runtime": 10.4273,
     "train_samples": 1030,
-    "train_samples_per_second": 102.281,
-    "train_steps_per_second": 3.322
+    "train_samples_per_second": 141.744,
+    "train_steps_per_second": 1.151
 }

trainer_state.json

@@ -3,113 +3,64 @@
   "best_model_checkpoint": null,
   "epoch": 2.0,
   "eval_steps": 500,
-  "global_step": 48,
+  "global_step": 12,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.041666666666666664,
-      "grad_norm": 6.875,
-      "learning_rate": 4.000000000000001e-06,
-      "loss": 3.886,
+      "epoch": 0.17,
+      "grad_norm": 5.802258301213437,
+      "learning_rate": 1e-05,
+      "loss": 3.714,
       "step": 1
     },
     {
-      "epoch": 0.20833333333333334,
-      "grad_norm": 5.15625,
-      "learning_rate": 2e-05,
-      "loss": 3.5195,
+      "epoch": 0.83,
+      "grad_norm": 4.228032304421419,
+      "learning_rate": 1.5877852522924733e-05,
+      "loss": 3.6492,
       "step": 5
     },
-    {
-      "epoch": 0.4166666666666667,
-      "grad_norm": 3.546875,
-      "learning_rate": 1.9340161087325483e-05,
-      "loss": 3.6848,
-      "step": 10
-    },
-    {
-      "epoch": 0.625,
-      "grad_norm": 3.0625,
-      "learning_rate": 1.744772182743782e-05,
-      "loss": 3.5055,
-      "step": 15
-    },
-    {
-      "epoch": 0.8333333333333334,
-      "grad_norm": 3.0,
-      "learning_rate": 1.4572423233046386e-05,
-      "loss": 3.4931,
-      "step": 20
-    },
     {
       "epoch": 1.0,
-      "eval_loss": 5.596227169036865,
-      "eval_runtime": 0.0423,
-      "eval_samples_per_second": 283.93,
-      "eval_steps_per_second": 47.322,
-      "step": 24
-    },
-    {
-      "epoch": 1.0416666666666667,
-      "grad_norm": 2.90625,
-      "learning_rate": 1.1093712083778748e-05,
-      "loss": 3.4722,
-      "step": 25
-    },
-    {
-      "epoch": 1.25,
-      "grad_norm": 2.828125,
-      "learning_rate": 7.470666176083193e-06,
-      "loss": 3.4108,
-      "step": 30
-    },
-    {
-      "epoch": 1.4583333333333333,
-      "grad_norm": 2.5,
-      "learning_rate": 4.181410844420473e-06,
-      "loss": 3.4187,
-      "step": 35
-    },
-    {
-      "epoch": 1.6666666666666665,
-      "grad_norm": 2.8125,
-      "learning_rate": 1.660021821101222e-06,
-      "loss": 3.4852,
-      "step": 40
+      "eval_loss": 5.712366104125977,
+      "eval_runtime": 0.0483,
+      "eval_samples_per_second": 248.688,
+      "eval_steps_per_second": 20.724,
+      "step": 6
     },
     {
-      "epoch": 1.875,
-      "grad_norm": 2.484375,
-      "learning_rate": 2.392412244407294e-07,
-      "loss": 3.4278,
-      "step": 45
+      "epoch": 1.67,
+      "grad_norm": 2.84117612385776,
+      "learning_rate": 1.9098300562505266e-06,
+      "loss": 3.457,
+      "step": 10
     },
     {
       "epoch": 2.0,
-      "eval_loss": 5.484647274017334,
-      "eval_runtime": 0.0423,
-      "eval_samples_per_second": 283.999,
-      "eval_steps_per_second": 47.333,
-      "step": 48
+      "eval_loss": 5.508264064788818,
+      "eval_runtime": 0.0486,
+      "eval_samples_per_second": 247.068,
+      "eval_steps_per_second": 20.589,
+      "step": 12
     },
     {
       "epoch": 2.0,
-      "step": 48,
-      "total_flos": 394600362541056.0,
-      "train_loss": 3.4869212259848914,
-      "train_runtime": 14.4504,
-      "train_samples_per_second": 102.281,
-      "train_steps_per_second": 3.322
+      "step": 12,
+      "total_flos": 36238786560.0,
+      "train_loss": 3.5401106079419455,
+      "train_runtime": 10.4273,
+      "train_samples_per_second": 141.744,
+      "train_steps_per_second": 1.151
     }
   ],
   "logging_steps": 5,
-  "max_steps": 48,
+  "max_steps": 12,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 1000000000,
-  "total_flos": 394600362541056.0,
+  "total_flos": 36238786560.0,
   "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2c9ebcd5bf35bdc52b59db2bc7861d16777727aab05faf5f6fe289107775724
-size 5048
+oid sha256:43e697c2bae2940ee6c5029cb760ba9a42f675be9a07073967082c9baeb458d4
+size 6072