Training in progress, step 500

README.md

@@ -19,7 +19,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [imone/Mistral_7B_with_EOT_token](https://huggingface.co/imone/Mistral_7B_with_EOT_token) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.7659
+- Loss: 0.1581
 
 ## Model description
 
@@ -39,12 +39,12 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 2e-05
-- train_batch_size: 4
+- train_batch_size: 16
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
 - num_devices: 8
-- total_train_batch_size: 32
+- total_train_batch_size: 128
 - total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
@@ -55,16 +55,16 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 0.4066        | 1.0   | 235  | 0.7733          |
-| 0.1901        | 2.0   | 470  | 0.6737          |
-| 0.068         | 3.0   | 705  | 0.6576          |
-| 0.0183        | 4.0   | 940  | 0.7072          |
-| 0.004         | 5.0   | 1175 | 0.7659          |
+| 0.6823        | 1.0   | 565  | 0.6283          |
+| 0.4922        | 2.0   | 1130 | 0.3859          |
+| 0.3003        | 3.0   | 1695 | 0.2350          |
+| 0.1776        | 4.0   | 2260 | 0.1633          |
+| 0.0793        | 5.0   | 2825 | 0.1581          |
 
 
 ### Framework versions
 
-- Transformers 4.38.2
+- Transformers 4.40.0
 - Pytorch 2.1.2+cu118
-- Datasets 2.16.1
-- Tokenizers 0.15.2
+- Datasets 2.18.0
+- Tokenizers 0.19.1

all_results.json

@@ -1,8 +1,9 @@
 {
     "epoch": 5.0,
-    "train_loss": 0.1592100728509274,
-    "train_runtime": 9397.2382,
-    "train_samples": 6599,
-    "train_samples_per_second": 3.994,
-    "train_steps_per_second": 0.125
+    "total_flos": 2365990109184000.0,
+    "train_loss": 0.3740393664457102,
+    "train_runtime": 65777.1761,
+    "train_samples": 83372,
+    "train_samples_per_second": 5.497,
+    "train_steps_per_second": 0.043
 }

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.0",
+  "transformers_version": "4.38.2",
   "use_cache": false,
   "vocab_size": 32002
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 32000,
-  "transformers_version": "4.38.2"
+  "transformers_version": "4.40.0"
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd14bffb7566339a201495e4e9353b15e3d92fb911ce4f4a3f5de1d45e188659
+oid sha256:23e38b93eb70a75bc1796a88024daae87822fa1048c276914b3345c1124fa371
 size 4943178720

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1980a6a3dfbd30201c129c13c1af5642aea406983e195f28bc4de851842240aa
+oid sha256:8587edd9d1965007b93ecb6e6421384db0ce99dfc1eaa7a6f08f8ea8d4dbe76d
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d31c69f76a6d11b6079f28272a99c7c48c8cf6ee791eb2b7bbad8df104021418
+oid sha256:59633e4e185330e9e5536ea22a6271ef1271c28f0017b294871f73c659a92a8f
 size 4540532728

runs/Apr21_15-02-12_n136-085-012/events.out.tfevents.1713683652.n136-085-012.31939.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:744f38fc7314a9c3307484c4e1214911946217eacc65996b359ea613ee38289c
+size 19630

runs/Apr21_17-29-09_n136-085-012/events.out.tfevents.1713691778.n136-085-012.82033.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52ae0e7260c05ddaf15131c576cfa8172db8f006052a30b7a5e053b91fc9f281
+size 25960

tokenizer.json

@@ -152,7 +152,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -1,8 +1,9 @@
 {
     "epoch": 5.0,
-    "train_loss": 0.1592100728509274,
-    "train_runtime": 9397.2382,
-    "train_samples": 6599,
-    "train_samples_per_second": 3.994,
-    "train_steps_per_second": 0.125
+    "total_flos": 2365990109184000.0,
+    "train_loss": 0.3740393664457102,
+    "train_runtime": 65777.1761,
+    "train_samples": 83372,
+    "train_samples_per_second": 5.497,
+    "train_steps_per_second": 0.043
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa332ee855a131766188374f885382875a0b64d355e2f53480c7a0d2aede4e5f
-size 6200
+oid sha256:f7823a59e69033859e4fa36c381894c53ecd9b05aaca1dfafc0ecc5dbf097f83
+size 6136