Model save

README.md

@@ -0,0 +1,70 @@
+---
+license: apache-2.0
+base_model: imone/Mistral_7B_with_EOT_token
+tags:
+- trl
+- sft
+- generated_from_trainer
+datasets:
+- generator
+model-index:
+- name: zephyr-7b-sft-full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-sft-full
+
+This model is a fine-tuned version of [imone/Mistral_7B_with_EOT_token](https://huggingface.co/imone/Mistral_7B_with_EOT_token) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0757
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 16
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 128
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 5
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 0.6546        | 1.0   | 59   | 0.4449          |
+| 0.4747        | 2.0   | 118  | 0.3181          |
+| 0.2246        | 3.0   | 177  | 0.1280          |
+| 0.1179        | 4.0   | 236  | 0.0868          |
+| 0.0755        | 5.0   | 295  | 0.0757          |
+
+
+### Framework versions
+
+- Transformers 4.38.2
+- Pytorch 2.1.2+cu118
+- Datasets 2.16.1
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 5.0,
+    "train_loss": 0.35338847313897087,
+    "train_runtime": 8245.0701,
+    "train_samples": 6599,
+    "train_samples_per_second": 4.561,
+    "train_steps_per_second": 0.036
+}

config.json

@@ -20,7 +20,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.0",
+  "transformers_version": "4.38.2",
   "use_cache": false,
   "vocab_size": 32002
 }

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 32000,
+  "transformers_version": "4.38.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:066b7e19b466a58cbdb32f37296e58694c72e117b7a5dabcf84cb1ac78954f65
+size 4943178720

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:657ecf192480f52a94c8bcb9ef088caa89e22c8c78b47133868548b47c2ac246
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5890f26d94791a70413345636c1e032afdb83a2944f2abf632d650d978913320
+size 4540532728

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483496960
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Apr19_20-47-51_n136-147-090/events.out.tfevents.1713531041.n136-147-090.546117.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7fd1a5eaab17c8bc9bf143025bffc12d686acd01b87268af7ebab8fdd7958081
-size 14118
+oid sha256:f7a80641419b976505969c6569383a0403f3e51f1674d6032e47c8d725022903
+size 19023

runs/Apr19_23-13-20_n136-147-090/events.out.tfevents.1713539677.n136-147-090.624072.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:103d2f07a9b60d9c050967f788645b92f92074197fb1237b300e79de5d609aa2
+size 9672

tokenizer.json

@@ -152,7 +152,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 5.0,
+    "train_loss": 0.35338847313897087,
+    "train_runtime": 8245.0701,
+    "train_samples": 6599,
+    "train_samples_per_second": 4.561,
+    "train_steps_per_second": 0.036
+}

trainer_state.json

@@ -0,0 +1,490 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.0,
+  "eval_steps": 500,
+  "global_step": 295,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02,
+      "grad_norm": 4.7237044668329,
+      "learning_rate": 6.666666666666667e-07,
+      "loss": 0.9545,
+      "step": 1
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 3.1773052856369324,
+      "learning_rate": 3.3333333333333333e-06,
+      "loss": 0.9326,
+      "step": 5
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 3.033793297452865,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.8956,
+      "step": 10
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 2.0855419235541013,
+      "learning_rate": 1e-05,
+      "loss": 0.8574,
+      "step": 15
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 2.0199202255230047,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 0.8222,
+      "step": 20
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 2.3471928271965106,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 0.8138,
+      "step": 25
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 2.600668692972917,
+      "learning_rate": 2e-05,
+      "loss": 0.7721,
+      "step": 30
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 2.4226542415206485,
+      "learning_rate": 1.9982437317643218e-05,
+      "loss": 0.7634,
+      "step": 35
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 2.850819690314508,
+      "learning_rate": 1.992981096013517e-05,
+      "loss": 0.7521,
+      "step": 40
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 2.5549789377169114,
+      "learning_rate": 1.984230577947597e-05,
+      "loss": 0.697,
+      "step": 45
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 2.4774565913803563,
+      "learning_rate": 1.972022914080411e-05,
+      "loss": 0.6735,
+      "step": 50
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 2.146208038921012,
+      "learning_rate": 1.9564009842765225e-05,
+      "loss": 0.6546,
+      "step": 55
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.4448932409286499,
+      "eval_runtime": 334.8516,
+      "eval_samples_per_second": 22.461,
+      "eval_steps_per_second": 0.352,
+      "step": 59
+    },
+    {
+      "epoch": 1.02,
+      "grad_norm": 3.923304638941212,
+      "learning_rate": 1.9374196611341212e-05,
+      "loss": 0.6271,
+      "step": 60
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 3.1379021048274596,
+      "learning_rate": 1.9151456172430186e-05,
+      "loss": 0.4188,
+      "step": 65
+    },
+    {
+      "epoch": 1.19,
+      "grad_norm": 2.478413663736477,
+      "learning_rate": 1.8896570909947477e-05,
+      "loss": 0.4197,
+      "step": 70
+    },
+    {
+      "epoch": 1.27,
+      "grad_norm": 3.1965773364967114,
+      "learning_rate": 1.8610436117673557e-05,
+      "loss": 0.3996,
+      "step": 75
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 1.8635605447479031,
+      "learning_rate": 1.829405685450202e-05,
+      "loss": 0.4087,
+      "step": 80
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 4.049894316503265,
+      "learning_rate": 1.7948544414133534e-05,
+      "loss": 0.4113,
+      "step": 85
+    },
+    {
+      "epoch": 1.53,
+      "grad_norm": 175.82031352565957,
+      "learning_rate": 1.7575112421616203e-05,
+      "loss": 0.7264,
+      "step": 90
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 97.72323752808755,
+      "learning_rate": 1.717507257044331e-05,
+      "loss": 0.7019,
+      "step": 95
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 10.439473368823062,
+      "learning_rate": 1.6749830015182106e-05,
+      "loss": 0.8288,
+      "step": 100
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 14.65204562703183,
+      "learning_rate": 1.6300878435817115e-05,
+      "loss": 0.622,
+      "step": 105
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 4.935916944271238,
+      "learning_rate": 1.5829794791144723e-05,
+      "loss": 0.5399,
+      "step": 110
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 3.1671885548837544,
+      "learning_rate": 1.533823377964791e-05,
+      "loss": 0.4747,
+      "step": 115
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 0.31807941198349,
+      "eval_runtime": 335.1235,
+      "eval_samples_per_second": 22.442,
+      "eval_steps_per_second": 0.352,
+      "step": 118
+    },
+    {
+      "epoch": 2.03,
+      "grad_norm": 2.7978128026277864,
+      "learning_rate": 1.482792202730745e-05,
+      "loss": 0.3959,
+      "step": 120
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 2.1385098508541693,
+      "learning_rate": 1.4300652022765207e-05,
+      "loss": 0.2884,
+      "step": 125
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 1.9703828411806736,
+      "learning_rate": 1.3758275821142382e-05,
+      "loss": 0.2745,
+      "step": 130
+    },
+    {
+      "epoch": 2.29,
+      "grad_norm": 1.870032223898805,
+      "learning_rate": 1.3202698538628376e-05,
+      "loss": 0.2539,
+      "step": 135
+    },
+    {
+      "epoch": 2.37,
+      "grad_norm": 7.255082480038806,
+      "learning_rate": 1.2635871660690677e-05,
+      "loss": 0.2588,
+      "step": 140
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 1.7733158231347383,
+      "learning_rate": 1.2059786187410984e-05,
+      "loss": 0.2564,
+      "step": 145
+    },
+    {
+      "epoch": 2.54,
+      "grad_norm": 1.9182852956652028,
+      "learning_rate": 1.1476465640024814e-05,
+      "loss": 0.2282,
+      "step": 150
+    },
+    {
+      "epoch": 2.63,
+      "grad_norm": 1.6601552566584281,
+      "learning_rate": 1.0887958953229349e-05,
+      "loss": 0.2323,
+      "step": 155
+    },
+    {
+      "epoch": 2.71,
+      "grad_norm": 1.4919318406362796,
+      "learning_rate": 1.0296333278225599e-05,
+      "loss": 0.2354,
+      "step": 160
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 1.5076718836100451,
+      "learning_rate": 9.703666721774403e-06,
+      "loss": 0.23,
+      "step": 165
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 1.703882565208849,
+      "learning_rate": 9.112041046770653e-06,
+      "loss": 0.2312,
+      "step": 170
+    },
+    {
+      "epoch": 2.97,
+      "grad_norm": 1.3815318382548532,
+      "learning_rate": 8.52353435997519e-06,
+      "loss": 0.2246,
+      "step": 175
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.12802040576934814,
+      "eval_runtime": 335.0335,
+      "eval_samples_per_second": 22.449,
+      "eval_steps_per_second": 0.352,
+      "step": 177
+    },
+    {
+      "epoch": 3.05,
+      "grad_norm": 1.340084619420041,
+      "learning_rate": 7.940213812589018e-06,
+      "loss": 0.1568,
+      "step": 180
+    },
+    {
+      "epoch": 3.14,
+      "grad_norm": 1.4363530293138602,
+      "learning_rate": 7.364128339309326e-06,
+      "loss": 0.1171,
+      "step": 185
+    },
+    {
+      "epoch": 3.22,
+      "grad_norm": 7.055735555930357,
+      "learning_rate": 6.797301461371626e-06,
+      "loss": 0.129,
+      "step": 190
+    },
+    {
+      "epoch": 3.31,
+      "grad_norm": 3.3324655424159606,
+      "learning_rate": 6.241724178857621e-06,
+      "loss": 0.15,
+      "step": 195
+    },
+    {
+      "epoch": 3.39,
+      "grad_norm": 2.4413759201237917,
+      "learning_rate": 5.699347977234799e-06,
+      "loss": 0.137,
+      "step": 200
+    },
+    {
+      "epoch": 3.47,
+      "grad_norm": 1.6373540256340913,
+      "learning_rate": 5.172077972692553e-06,
+      "loss": 0.1218,
+      "step": 205
+    },
+    {
+      "epoch": 3.56,
+      "grad_norm": 1.516908679115731,
+      "learning_rate": 4.661766220352098e-06,
+      "loss": 0.1242,
+      "step": 210
+    },
+    {
+      "epoch": 3.64,
+      "grad_norm": 1.4120761150066543,
+      "learning_rate": 4.170205208855281e-06,
+      "loss": 0.1163,
+      "step": 215
+    },
+    {
+      "epoch": 3.73,
+      "grad_norm": 1.2013256002801418,
+      "learning_rate": 3.6991215641828903e-06,
+      "loss": 0.1212,
+      "step": 220
+    },
+    {
+      "epoch": 3.81,
+      "grad_norm": 1.2326050504759452,
+      "learning_rate": 3.250169984817897e-06,
+      "loss": 0.1261,
+      "step": 225
+    },
+    {
+      "epoch": 3.9,
+      "grad_norm": 1.189664724350827,
+      "learning_rate": 2.8249274295566863e-06,
+      "loss": 0.1206,
+      "step": 230
+    },
+    {
+      "epoch": 3.98,
+      "grad_norm": 1.1432302198824411,
+      "learning_rate": 2.424887578383799e-06,
+      "loss": 0.1179,
+      "step": 235
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 0.08719707280397415,
+      "eval_runtime": 336.1886,
+      "eval_samples_per_second": 22.371,
+      "eval_steps_per_second": 0.351,
+      "step": 236
+    },
+    {
+      "epoch": 4.07,
+      "grad_norm": 1.2963972162778623,
+      "learning_rate": 2.0514555858664663e-06,
+      "loss": 0.0919,
+      "step": 240
+    },
+    {
+      "epoch": 4.15,
+      "grad_norm": 1.2710485222195504,
+      "learning_rate": 1.7059431454979825e-06,
+      "loss": 0.0833,
+      "step": 245
+    },
+    {
+      "epoch": 4.24,
+      "grad_norm": 1.142352876486223,
+      "learning_rate": 1.3895638823264447e-06,
+      "loss": 0.0861,
+      "step": 250
+    },
+    {
+      "epoch": 4.32,
+      "grad_norm": 0.9500339112967238,
+      "learning_rate": 1.1034290900525279e-06,
+      "loss": 0.0749,
+      "step": 255
+    },
+    {
+      "epoch": 4.41,
+      "grad_norm": 0.9628764967033802,
+      "learning_rate": 8.485438275698154e-07,
+      "loss": 0.0847,
+      "step": 260
+    },
+    {
+      "epoch": 4.49,
+      "grad_norm": 1.0664629678640016,
+      "learning_rate": 6.258033886587911e-07,
+      "loss": 0.0832,
+      "step": 265
+    },
+    {
+      "epoch": 4.58,
+      "grad_norm": 0.9360345524617034,
+      "learning_rate": 4.359901572347758e-07,
+      "loss": 0.0819,
+      "step": 270
+    },
+    {
+      "epoch": 4.66,
+      "grad_norm": 0.9019649784225472,
+      "learning_rate": 2.7977085919589253e-07,
+      "loss": 0.081,
+      "step": 275
+    },
+    {
+      "epoch": 4.75,
+      "grad_norm": 0.9156461143807438,
+      "learning_rate": 1.5769422052403172e-07,
+      "loss": 0.0777,
+      "step": 280
+    },
+    {
+      "epoch": 4.83,
+      "grad_norm": 0.9560172751872731,
+      "learning_rate": 7.018903986483083e-08,
+      "loss": 0.0804,
+      "step": 285
+    },
+    {
+      "epoch": 4.92,
+      "grad_norm": 1.0000085208264247,
+      "learning_rate": 1.7562682356786488e-08,
+      "loss": 0.0842,
+      "step": 290
+    },
+    {
+      "epoch": 5.0,
+      "grad_norm": 0.8561802846895736,
+      "learning_rate": 0.0,
+      "loss": 0.0755,
+      "step": 295
+    },
+    {
+      "epoch": 5.0,
+      "eval_loss": 0.07610141485929489,
+      "eval_runtime": 335.8094,
+      "eval_samples_per_second": 22.397,
+      "eval_steps_per_second": 0.351,
+      "step": 295
+    },
+    {
+      "epoch": 5.0,
+      "step": 295,
+      "total_flos": 247067993702400.0,
+      "train_loss": 0.35338847313897087,
+      "train_runtime": 8245.0701,
+      "train_samples_per_second": 4.561,
+      "train_steps_per_second": 0.036
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 295,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 100,
+  "total_flos": 247067993702400.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ed76c4686c16785ebd81560b9c9a3c4a03f96915a52041230fccff8e48e404f
-size 6200
+oid sha256:5f3102b5342751991de00e200dae83b60392b6184ebcde42145c83686df6bce6
+size 6136