Model save

README.md

@@ -0,0 +1,66 @@
+---
+license: apache-2.0
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- trl
+- sft
+- generated_from_trainer
+datasets:
+- generator
+model-index:
+- name: mistralai-7b-sft-full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistralai-7b-sft-full
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.9352
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 16
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 128
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 0.91          | 1.0   | 1089 | 0.9352          |
+
+
+### Framework versions
+
+- Transformers 4.36.2
+- Pytorch 2.1.2+cu118
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,13 @@
+{
+    "epoch": 1.0,
+    "eval_loss": 0.9352424740791321,
+    "eval_runtime": 317.7032,
+    "eval_samples": 23091,
+    "eval_samples_per_second": 48.526,
+    "eval_steps_per_second": 0.759,
+    "train_loss": 0.9776419559895609,
+    "train_runtime": 12679.62,
+    "train_samples": 207722,
+    "train_samples_per_second": 10.987,
+    "train_steps_per_second": 0.086
+}

eval_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "eval_loss": 0.9352424740791321,
+    "eval_runtime": 317.7032,
+    "eval_samples": 23091,
+    "eval_samples_per_second": 48.526,
+    "eval_steps_per_second": 0.759
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67dbe91f7f0a36f0ef0fd1eaa3019a9d490840e97fc1fbdc9981738956b0b5a6
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:111330f9ad7efb8465dcef2c090b7ffa2e041da6caa8f3389ea8bc389ade6956
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f33301d8111f3dde24b105f941f06ce0ee96140e78778776050293281fdccc4
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Feb21_21-28-24_tf3/events.out.tfevents.1708539253.tf3.3447762.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b67c4b5345f2ca0d780f776425c05fa567b238c57ee966d9bb522f8341fecc44
-size 35910
+oid sha256:6e1a64801e1bc37ce7786fe90f4095cd4b4294383d9a6b6d7c6cc795d6b2217d
+size 39204

runs/Feb21_21-28-24_tf3/events.out.tfevents.1708552250.tf3.3447762.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6729a6308246c6583578e6051958e4200405c59778e00d083f539040935eade4
+size 359

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.9776419559895609,
+    "train_runtime": 12679.62,
+    "train_samples": 207722,
+    "train_samples_per_second": 10.987,
+    "train_steps_per_second": 0.086
+}

trainer_state.json

@@ -0,0 +1,1346 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1089,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.8348623853211012e-07,
+      "loss": 1.6866,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.174311926605506e-07,
+      "loss": 1.6643,
+      "step": 5
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 1.8348623853211011e-06,
+      "loss": 1.4006,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.7522935779816517e-06,
+      "loss": 1.2478,
+      "step": 15
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 3.6697247706422022e-06,
+      "loss": 1.1691,
+      "step": 20
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 4.587155963302753e-06,
+      "loss": 1.1192,
+      "step": 25
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 5.504587155963303e-06,
+      "loss": 1.0745,
+      "step": 30
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 6.422018348623854e-06,
+      "loss": 1.0693,
+      "step": 35
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 7.3394495412844045e-06,
+      "loss": 1.06,
+      "step": 40
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 8.256880733944956e-06,
+      "loss": 1.0413,
+      "step": 45
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 9.174311926605506e-06,
+      "loss": 1.0245,
+      "step": 50
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.0091743119266055e-05,
+      "loss": 1.0305,
+      "step": 55
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.1009174311926607e-05,
+      "loss": 1.0137,
+      "step": 60
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.1926605504587156e-05,
+      "loss": 0.9991,
+      "step": 65
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.2844036697247708e-05,
+      "loss": 1.0178,
+      "step": 70
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.3761467889908258e-05,
+      "loss": 0.9941,
+      "step": 75
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.4678899082568809e-05,
+      "loss": 1.0157,
+      "step": 80
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.559633027522936e-05,
+      "loss": 1.0254,
+      "step": 85
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.6513761467889912e-05,
+      "loss": 1.0103,
+      "step": 90
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.743119266055046e-05,
+      "loss": 1.0006,
+      "step": 95
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.834862385321101e-05,
+      "loss": 0.9974,
+      "step": 100
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.9266055045871563e-05,
+      "loss": 1.0041,
+      "step": 105
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.999994861726391e-05,
+      "loss": 1.0137,
+      "step": 110
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.99981502769439e-05,
+      "loss": 1.0078,
+      "step": 115
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.999378332783191e-05,
+      "loss": 1.0148,
+      "step": 120
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.9986848891833846e-05,
+      "loss": 1.0185,
+      "step": 125
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.997734875046456e-05,
+      "loss": 1.0105,
+      "step": 130
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9965285344390185e-05,
+      "loss": 1.0015,
+      "step": 135
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.9950661772801062e-05,
+      "loss": 1.0084,
+      "step": 140
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.9933481792615583e-05,
+      "loss": 1.0053,
+      "step": 145
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.9913749817514963e-05,
+      "loss": 1.0154,
+      "step": 150
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.9891470916809362e-05,
+      "loss": 1.0014,
+      "step": 155
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.986665081413552e-05,
+      "loss": 1.0092,
+      "step": 160
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.98392958859863e-05,
+      "loss": 1.0185,
+      "step": 165
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.980941316007253e-05,
+      "loss": 1.0176,
+      "step": 170
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.9777010313517517e-05,
+      "loss": 1.0089,
+      "step": 175
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.974209567088473e-05,
+      "loss": 1.0228,
+      "step": 180
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.9704678202039148e-05,
+      "loss": 1.0189,
+      "step": 185
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.966476751984283e-05,
+      "loss": 0.9983,
+      "step": 190
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.962237387768529e-05,
+      "loss": 1.0197,
+      "step": 195
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.9577508166849308e-05,
+      "loss": 1.0004,
+      "step": 200
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.9530181913712875e-05,
+      "loss": 0.9991,
+      "step": 205
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.9480407276787968e-05,
+      "loss": 1.0092,
+      "step": 210
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.942819704359693e-05,
+      "loss": 0.9886,
+      "step": 215
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.9373564627387243e-05,
+      "loss": 1.0196,
+      "step": 220
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.9316524063685544e-05,
+      "loss": 1.0169,
+      "step": 225
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.92570900066918e-05,
+      "loss": 1.0254,
+      "step": 230
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.919527772551451e-05,
+      "loss": 1.0024,
+      "step": 235
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.9131103100247934e-05,
+      "loss": 1.0204,
+      "step": 240
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.9064582617892383e-05,
+      "loss": 1.0196,
+      "step": 245
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.8995733368118556e-05,
+      "loss": 1.0135,
+      "step": 250
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.8924573038877062e-05,
+      "loss": 1.0254,
+      "step": 255
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.8851119911854236e-05,
+      "loss": 1.0024,
+      "step": 260
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.877539285777543e-05,
+      "loss": 1.0066,
+      "step": 265
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.8697411331556958e-05,
+      "loss": 0.9814,
+      "step": 270
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.861719536730795e-05,
+      "loss": 0.9967,
+      "step": 275
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.853476557318346e-05,
+      "loss": 1.0078,
+      "step": 280
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.8450143126090015e-05,
+      "loss": 1.0156,
+      "step": 285
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.8363349766245108e-05,
+      "loss": 1.0144,
+      "step": 290
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.8274407791591966e-05,
+      "loss": 1.0124,
+      "step": 295
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.8183340052071e-05,
+      "loss": 0.9986,
+      "step": 300
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.8090169943749477e-05,
+      "loss": 0.9987,
+      "step": 305
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.7994921402810862e-05,
+      "loss": 1.0137,
+      "step": 310
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.7897618899405423e-05,
+      "loss": 1.0134,
+      "step": 315
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.7798287431363644e-05,
+      "loss": 1.0011,
+      "step": 320
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.769695251777406e-05,
+      "loss": 0.9996,
+      "step": 325
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.75936401924272e-05,
+      "loss": 1.0123,
+      "step": 330
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.748837699712728e-05,
+      "loss": 1.003,
+      "step": 335
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.738118997487341e-05,
+      "loss": 1.008,
+      "step": 340
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.7272106662911972e-05,
+      "loss": 0.9967,
+      "step": 345
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.7161155085662144e-05,
+      "loss": 0.9983,
+      "step": 350
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.704836374751612e-05,
+      "loss": 0.9894,
+      "step": 355
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.693376162551613e-05,
+      "loss": 0.9987,
+      "step": 360
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.6817378161909995e-05,
+      "loss": 0.9949,
+      "step": 365
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.6699243256587156e-05,
+      "loss": 0.9991,
+      "step": 370
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.657938725939713e-05,
+      "loss": 0.997,
+      "step": 375
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.6457840962352403e-05,
+      "loss": 0.9866,
+      "step": 380
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.63346355917177e-05,
+      "loss": 0.9847,
+      "step": 385
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.6209802799987674e-05,
+      "loss": 0.9972,
+      "step": 390
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.6083374657755132e-05,
+      "loss": 0.9712,
+      "step": 395
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.5955383645471828e-05,
+      "loss": 0.9911,
+      "step": 400
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.5825862645103962e-05,
+      "loss": 0.9882,
+      "step": 405
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.569484493168452e-05,
+      "loss": 0.9958,
+      "step": 410
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.556236416476465e-05,
+      "loss": 0.9738,
+      "step": 415
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.5428454379766223e-05,
+      "loss": 0.9654,
+      "step": 420
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.5293149979237875e-05,
+      "loss": 0.987,
+      "step": 425
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.5156485724016672e-05,
+      "loss": 0.9919,
+      "step": 430
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.5018496724297778e-05,
+      "loss": 1.011,
+      "step": 435
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.4879218430614346e-05,
+      "loss": 0.9793,
+      "step": 440
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.4738686624729987e-05,
+      "loss": 0.9981,
+      "step": 445
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.4596937410446117e-05,
+      "loss": 1.0022,
+      "step": 450
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.4454007204326592e-05,
+      "loss": 0.9988,
+      "step": 455
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.4309932726342007e-05,
+      "loss": 0.9803,
+      "step": 460
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.4164750990435991e-05,
+      "loss": 0.9845,
+      "step": 465
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.4018499295016057e-05,
+      "loss": 0.9902,
+      "step": 470
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.3871215213371284e-05,
+      "loss": 0.9816,
+      "step": 475
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.3722936584019453e-05,
+      "loss": 0.9785,
+      "step": 480
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.3573701500986012e-05,
+      "loss": 0.9802,
+      "step": 485
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.342354830401738e-05,
+      "loss": 0.986,
+      "step": 490
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.327251556873117e-05,
+      "loss": 0.9719,
+      "step": 495
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.3120642096705773e-05,
+      "loss": 0.9845,
+      "step": 500
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.2967966905511906e-05,
+      "loss": 0.989,
+      "step": 505
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.2814529218688688e-05,
+      "loss": 0.9675,
+      "step": 510
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.2660368455666752e-05,
+      "loss": 0.9767,
+      "step": 515
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.2505524221641097e-05,
+      "loss": 0.9966,
+      "step": 520
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.2350036297396153e-05,
+      "loss": 0.9569,
+      "step": 525
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.2193944629085778e-05,
+      "loss": 0.9773,
+      "step": 530
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.2037289317970757e-05,
+      "loss": 0.9683,
+      "step": 535
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.1880110610116438e-05,
+      "loss": 0.9737,
+      "step": 540
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.172244888605319e-05,
+      "loss": 0.9781,
+      "step": 545
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.156434465040231e-05,
+      "loss": 0.9362,
+      "step": 550
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.140583852147003e-05,
+      "loss": 0.9604,
+      "step": 555
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.1246971220812348e-05,
+      "loss": 0.9693,
+      "step": 560
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 1.108778356277331e-05,
+      "loss": 0.9694,
+      "step": 565
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 1.0928316443999462e-05,
+      "loss": 0.9649,
+      "step": 570
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 1.0768610832933169e-05,
+      "loss": 0.9636,
+      "step": 575
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 1.0608707759287454e-05,
+      "loss": 0.9681,
+      "step": 580
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 1.044864830350515e-05,
+      "loss": 0.976,
+      "step": 585
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 1.028847358620497e-05,
+      "loss": 0.9476,
+      "step": 590
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 1.0128224757617272e-05,
+      "loss": 0.9634,
+      "step": 595
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 9.967942987012243e-06,
+      "loss": 0.9747,
+      "step": 600
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 9.80766945212313e-06,
+      "loss": 0.9569,
+      "step": 605
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 9.647445328567368e-06,
+      "loss": 0.9764,
+      "step": 610
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 9.48731177926821e-06,
+      "loss": 0.9533,
+      "step": 615
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 9.327309943879604e-06,
+      "loss": 0.9652,
+      "step": 620
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 9.167480928217108e-06,
+      "loss": 0.9585,
+      "step": 625
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 9.007865793697425e-06,
+      "loss": 0.9581,
+      "step": 630
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 8.848505546789407e-06,
+      "loss": 0.9778,
+      "step": 635
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 8.689441128479134e-06,
+      "loss": 0.9708,
+      "step": 640
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 8.530713403751822e-06,
+      "loss": 0.9608,
+      "step": 645
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 8.372363151093302e-06,
+      "loss": 0.9606,
+      "step": 650
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 8.214431052013636e-06,
+      "loss": 0.9628,
+      "step": 655
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 8.056957680595733e-06,
+      "loss": 0.9792,
+      "step": 660
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 7.899983493071506e-06,
+      "loss": 0.9443,
+      "step": 665
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 7.74354881742834e-06,
+      "loss": 0.9482,
+      "step": 670
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 7.587693843048475e-06,
+      "loss": 0.9664,
+      "step": 675
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 7.432458610384037e-06,
+      "loss": 0.9551,
+      "step": 680
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 7.27788300067029e-06,
+      "loss": 0.9468,
+      "step": 685
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 7.124006725679828e-06,
+      "loss": 0.9468,
+      "step": 690
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 6.97086931752028e-06,
+      "loss": 0.9351,
+      "step": 695
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 6.818510118478172e-06,
+      "loss": 0.9474,
+      "step": 700
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 6.666968270911585e-06,
+      "loss": 0.9416,
+      "step": 705
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 6.5162827071941194e-06,
+      "loss": 0.9444,
+      "step": 710
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 6.366492139712886e-06,
+      "loss": 0.9407,
+      "step": 715
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 6.217635050922923e-06,
+      "loss": 0.9594,
+      "step": 720
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 6.069749683460765e-06,
+      "loss": 0.9441,
+      "step": 725
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.922874030319567e-06,
+      "loss": 0.9318,
+      "step": 730
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.7770458250884044e-06,
+      "loss": 0.9424,
+      "step": 735
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 5.6323025322581694e-06,
+      "loss": 0.9462,
+      "step": 740
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 5.488681337596653e-06,
+      "loss": 0.9335,
+      "step": 745
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 5.346219138595215e-06,
+      "loss": 0.9482,
+      "step": 750
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 5.2049525349894625e-06,
+      "loss": 0.9221,
+      "step": 755
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 5.064917819356532e-06,
+      "loss": 0.9371,
+      "step": 760
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.92615096779118e-06,
+      "loss": 0.9476,
+      "step": 765
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.788687630663232e-06,
+      "loss": 0.9509,
+      "step": 770
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.652563123458703e-06,
+      "loss": 0.9403,
+      "step": 775
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 4.517812417706967e-06,
+      "loss": 0.9561,
+      "step": 780
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 4.3844701319962525e-06,
+      "loss": 0.9271,
+      "step": 785
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 4.252570523079852e-06,
+      "loss": 0.9379,
+      "step": 790
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 4.12214747707527e-06,
+      "loss": 0.9291,
+      "step": 795
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 3.993234500758597e-06,
+      "loss": 0.9394,
+      "step": 800
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.865864712956336e-06,
+      "loss": 0.9482,
+      "step": 805
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.740070836036893e-06,
+      "loss": 0.9267,
+      "step": 810
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.6158851875039458e-06,
+      "loss": 0.9338,
+      "step": 815
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.493339671693765e-06,
+      "loss": 0.943,
+      "step": 820
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 3.372465771578771e-06,
+      "loss": 0.9358,
+      "step": 825
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 3.2532945406792573e-06,
+      "loss": 0.9317,
+      "step": 830
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 3.135856595085498e-06,
+      "loss": 0.9381,
+      "step": 835
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 3.02018210559221e-06,
+      "loss": 0.9284,
+      "step": 840
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.9063007899474214e-06,
+      "loss": 0.9266,
+      "step": 845
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.794241905217753e-06,
+      "loss": 0.9358,
+      "step": 850
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.6840342402719867e-06,
+      "loss": 0.9349,
+      "step": 855
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.5757061083850153e-06,
+      "loss": 0.9377,
+      "step": 860
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.469285339963892e-06,
+      "loss": 0.9396,
+      "step": 865
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.3647992753979698e-06,
+      "loss": 0.922,
+      "step": 870
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.2622747580349313e-06,
+      "loss": 0.9232,
+      "step": 875
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 2.1617381272845174e-06,
+      "loss": 0.9269,
+      "step": 880
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 2.063215211851678e-06,
+      "loss": 0.9323,
+      "step": 885
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.9667313231009955e-06,
+      "loss": 0.9264,
+      "step": 890
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.8723112485539741e-06,
+      "loss": 0.9233,
+      "step": 895
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.7799792455209019e-06,
+      "loss": 0.9325,
+      "step": 900
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.6897590348689607e-06,
+      "loss": 0.9426,
+      "step": 905
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.601673794928127e-06,
+      "loss": 0.9276,
+      "step": 910
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.5157461555364772e-06,
+      "loss": 0.9373,
+      "step": 915
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.4319981922263636e-06,
+      "loss": 0.9341,
+      "step": 920
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.350451420553065e-06,
+      "loss": 0.9356,
+      "step": 925
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.271126790567223e-06,
+      "loss": 0.9189,
+      "step": 930
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.19404468143262e-06,
+      "loss": 0.9093,
+      "step": 935
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.119224896190595e-06,
+      "loss": 0.9331,
+      "step": 940
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 1.0466866566724698e-06,
+      "loss": 0.9154,
+      "step": 945
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 9.764485985613092e-07,
+      "loss": 0.9145,
+      "step": 950
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 9.085287666042508e-07,
+      "loss": 0.9264,
+      "step": 955
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 8.429446099766614e-07,
+      "loss": 0.9288,
+      "step": 960
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 7.797129777992951e-07,
+      "loss": 0.9188,
+      "step": 965
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 7.188501148096117e-07,
+      "loss": 0.9216,
+      "step": 970
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 6.60371657188369e-07,
+      "loss": 0.9218,
+      "step": 975
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 6.042926285425577e-07,
+      "loss": 0.9174,
+      "step": 980
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.506274360457087e-07,
+      "loss": 0.9052,
+      "step": 985
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.993898667365671e-07,
+      "loss": 0.9303,
+      "step": 990
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.505930839770967e-07,
+      "loss": 0.9262,
+      "step": 995
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.0424962407068167e-07,
+      "loss": 0.9285,
+      "step": 1000
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.603713930414676e-07,
+      "loss": 0.9216,
+      "step": 1005
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.189696635755868e-07,
+      "loss": 0.9181,
+      "step": 1010
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 2.8005507212512164e-07,
+      "loss": 0.9161,
+      "step": 1015
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.436376161755005e-07,
+      "loss": 0.9286,
+      "step": 1020
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.097266516770713e-07,
+      "loss": 0.9329,
+      "step": 1025
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.7833089064146825e-07,
+      "loss": 0.9304,
+      "step": 1030
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.494583989034326e-07,
+      "loss": 0.9353,
+      "step": 1035
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.231165940486234e-07,
+      "loss": 0.9119,
+      "step": 1040
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 9.931224350798185e-08,
+      "loss": 0.918,
+      "step": 1045
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 7.805146281912202e-08,
+      "loss": 0.914,
+      "step": 1050
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 5.933971405519656e-08,
+      "loss": 0.9287,
+      "step": 1055
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 4.318180442164588e-08,
+      "loss": 0.9122,
+      "step": 1060
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.9581885021181534e-08,
+      "loss": 0.9115,
+      "step": 1065
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.854344978733824e-08,
+      "loss": 0.9131,
+      "step": 1070
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0069334586854106e-08,
+      "loss": 0.9036,
+      "step": 1075
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.161716491105639e-09,
+      "loss": 0.9243,
+      "step": 1080
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 8.221132168073631e-10,
+      "loss": 0.91,
+      "step": 1085
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.9352424740791321,
+      "eval_runtime": 319.1826,
+      "eval_samples_per_second": 48.302,
+      "eval_steps_per_second": 0.755,
+      "step": 1089
+    },
+    {
+      "epoch": 1.0,
+      "step": 1089,
+      "total_flos": 456028890071040.0,
+      "train_loss": 0.9776419559895609,
+      "train_runtime": 12679.62,
+      "train_samples_per_second": 10.987,
+      "train_steps_per_second": 0.086
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1089,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 456028890071040.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}