Upload folder using huggingface_hub

README.md

@@ -0,0 +1,73 @@
+---
+license: other
+base_model: meta-llama/Meta-Llama-3-8B
+tags:
+- llama-factory
+- full
+- generated_from_trainer
+model-index:
+- name: C020_random_sample_llama3-8b-base_pretrain_20240505_135320
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# C020_random_sample_llama3-8b-base_pretrain_20240505_135320
+
+This model is a fine-tuned version of [/data/pro-align/progressalign/shared_storage/downloaded_models/llama3-8b-base](https://huggingface.co//data/pro-align/progressalign/shared_storage/downloaded_models/llama3-8b-base) on the C020_random_sample_data dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.9418
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1.5e-05
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 32
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: polynomial
+- lr_scheduler_warmup_steps: 20
+- num_epochs: 4.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 1.9087        | 0.4032 | 200  | 1.9717          |
+| 1.8752        | 0.8065 | 400  | 1.9418          |
+| 1.6383        | 1.2097 | 600  | 1.9440          |
+| 1.7073        | 1.6129 | 800  | 1.9435          |
+| 1.6699        | 2.0161 | 1000 | 1.9428          |
+| 1.7212        | 2.4194 | 1200 | 1.9445          |
+| 1.7346        | 2.8226 | 1400 | 1.9443          |
+| 1.7028        | 3.2258 | 1600 | 1.9448          |
+| 1.7383        | 3.6290 | 1800 | 1.9450          |
+
+
+### Framework versions
+
+- Transformers 4.40.1
+- Pytorch 2.3.0
+- Datasets 2.19.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,13 @@
+{
+    "epoch": 4.0,
+    "eval_loss": 1.9417628049850464,
+    "eval_runtime": 91.2862,
+    "eval_samples_per_second": 19.291,
+    "eval_steps_per_second": 0.307,
+    "perplexity": 6.971028709988671,
+    "total_flos": 414362338590720.0,
+    "train_loss": 1.7886004945443523,
+    "train_runtime": 13163.6015,
+    "train_samples_per_second": 4.815,
+    "train_steps_per_second": 0.151
+}

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "/data/pro-align/progressalign/shared_storage/downloaded_models/llama3-8b-base",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.40.1",
+  "use_cache": false,
+  "vocab_size": 128256
+}

eval_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 4.0,
+    "eval_loss": 1.9417628049850464,
+    "eval_runtime": 91.2862,
+    "eval_samples_per_second": 19.291,
+    "eval_steps_per_second": 0.307,
+    "perplexity": 6.971028709988671
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "transformers_version": "4.40.1"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e17d7e5ff8a6ad14a2551dd4fd600fb03872c570f92aa3aff029288736a91c09
+size 4976698592

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a773298e59fe1efe8b1673ca45db2e530fe7739ade128c2f361f61721c2e1ec
+size 4999802616

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b583d3ba1037169e8b3d278521b1e87c048f5ec79e1e081250a582fffa0cdaf3
+size 4915916080

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14e68d21422159f81a02c910fb5c7987e60f5f1e43f607ad09bc10ac8c793d76
+size 1168138808

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16060522496
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|end_of_text|>"
+}

tokenizer_config.json

@@ -0,0 +1,2065 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% set system_message = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '### Instruction:\\n' + content + '\\n\\n### Response:\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|end_of_text|>' + '\\n\\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|end_of_text|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 4.0,
+    "total_flos": 414362338590720.0,
+    "train_loss": 1.7886004945443523,
+    "train_runtime": 13163.6015,
+    "train_samples_per_second": 4.815,
+    "train_steps_per_second": 0.151
+}

trainer_state.json

@@ -0,0 +1,2881 @@
+{
+  "best_metric": 1.9417628049850464,
+  "best_model_checkpoint": "./output/training_results/C020_random_sample_llama3-8b-base_pretrain_20240505_135320/checkpoint-400",
+  "epoch": 4.0,
+  "eval_steps": 200,
+  "global_step": 1984,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0020161290322580645,
+      "grad_norm": 1.862664305700109,
+      "learning_rate": 7.5e-07,
+      "loss": 2.0502,
+      "step": 1
+    },
+    {
+      "epoch": 0.010080645161290322,
+      "grad_norm": 1.7486282966194167,
+      "learning_rate": 3.75e-06,
+      "loss": 2.0591,
+      "step": 5
+    },
+    {
+      "epoch": 0.020161290322580645,
+      "grad_norm": 1.5758042428174392,
+      "learning_rate": 6.750000000000001e-06,
+      "loss": 1.9721,
+      "step": 10
+    },
+    {
+      "epoch": 0.03024193548387097,
+      "grad_norm": 1.8438408614274,
+      "learning_rate": 9.75e-06,
+      "loss": 2.0137,
+      "step": 15
+    },
+    {
+      "epoch": 0.04032258064516129,
+      "grad_norm": 1.8618630610677624,
+      "learning_rate": 1.3500000000000001e-05,
+      "loss": 2.058,
+      "step": 20
+    },
+    {
+      "epoch": 0.05040322580645161,
+      "grad_norm": 2.12580198675208,
+      "learning_rate": 1.47507132047542e-05,
+      "loss": 2.0324,
+      "step": 25
+    },
+    {
+      "epoch": 0.06048387096774194,
+      "grad_norm": 1.4836280728067701,
+      "learning_rate": 1.4424209058252117e-05,
+      "loss": 1.9982,
+      "step": 30
+    },
+    {
+      "epoch": 0.07056451612903226,
+      "grad_norm": 1.6623583118858967,
+      "learning_rate": 1.4025354123166567e-05,
+      "loss": 1.8959,
+      "step": 35
+    },
+    {
+      "epoch": 0.08064516129032258,
+      "grad_norm": 1.5916601844779366,
+      "learning_rate": 1.3636585978219431e-05,
+      "loss": 2.0492,
+      "step": 40
+    },
+    {
+      "epoch": 0.0907258064516129,
+      "grad_norm": 1.7125711612090788,
+      "learning_rate": 1.3257674460153073e-05,
+      "loss": 1.9655,
+      "step": 45
+    },
+    {
+      "epoch": 0.10080645161290322,
+      "grad_norm": 1.632995352775119,
+      "learning_rate": 1.288839408598928e-05,
+      "loss": 2.0569,
+      "step": 50
+    },
+    {
+      "epoch": 0.11088709677419355,
+      "grad_norm": 1.9174014047156116,
+      "learning_rate": 1.2528523969541332e-05,
+      "loss": 1.9287,
+      "step": 55
+    },
+    {
+      "epoch": 0.12096774193548387,
+      "grad_norm": 1.867558573747783,
+      "learning_rate": 1.2177847739205852e-05,
+      "loss": 1.9894,
+      "step": 60
+    },
+    {
+      "epoch": 0.1310483870967742,
+      "grad_norm": 1.567626378627341,
+      "learning_rate": 1.1836153457018077e-05,
+      "loss": 2.0005,
+      "step": 65
+    },
+    {
+      "epoch": 0.14112903225806453,
+      "grad_norm": 1.3955216806991564,
+      "learning_rate": 1.1569125393187758e-05,
+      "loss": 2.0943,
+      "step": 70
+    },
+    {
+      "epoch": 0.15120967741935484,
+      "grad_norm": 1.4220083911016554,
+      "learning_rate": 1.1243078381650268e-05,
+      "loss": 1.9521,
+      "step": 75
+    },
+    {
+      "epoch": 0.16129032258064516,
+      "grad_norm": 1.5790411664333717,
+      "learning_rate": 1.0925442805236739e-05,
+      "loss": 2.0206,
+      "step": 80
+    },
+    {
+      "epoch": 0.17137096774193547,
+      "grad_norm": 1.3900088055150563,
+      "learning_rate": 1.0616022857385877e-05,
+      "loss": 1.9713,
+      "step": 85
+    },
+    {
+      "epoch": 0.1814516129032258,
+      "grad_norm": 1.3384719686058493,
+      "learning_rate": 1.0314626793719932e-05,
+      "loss": 1.925,
+      "step": 90
+    },
+    {
+      "epoch": 0.19153225806451613,
+      "grad_norm": 1.6023250792595878,
+      "learning_rate": 1.0021066858113034e-05,
+      "loss": 2.0494,
+      "step": 95
+    },
+    {
+      "epoch": 0.20161290322580644,
+      "grad_norm": 1.4194138427481093,
+      "learning_rate": 9.735159209915871e-06,
+      "loss": 2.0098,
+      "step": 100
+    },
+    {
+      "epoch": 0.21169354838709678,
+      "grad_norm": 1.2578428133053623,
+      "learning_rate": 9.456723852321558e-06,
+      "loss": 1.9048,
+      "step": 105
+    },
+    {
+      "epoch": 0.2217741935483871,
+      "grad_norm": 1.2538874670770739,
+      "learning_rate": 9.239237068300538e-06,
+      "loss": 1.9968,
+      "step": 110
+    },
+    {
+      "epoch": 0.2318548387096774,
+      "grad_norm": 1.39475982271136,
+      "learning_rate": 8.973810241991637e-06,
+      "loss": 2.0145,
+      "step": 115
+    },
+    {
+      "epoch": 0.24193548387096775,
+      "grad_norm": 1.2474176211653318,
+      "learning_rate": 8.715371568554332e-06,
+      "loss": 2.0293,
+      "step": 120
+    },
+    {
+      "epoch": 0.25201612903225806,
+      "grad_norm": 1.4147919193342449,
+      "learning_rate": 8.463755024156938e-06,
+      "loss": 1.9515,
+      "step": 125
+    },
+    {
+      "epoch": 0.2620967741935484,
+      "grad_norm": 1.2583008452280033,
+      "learning_rate": 8.218798100385521e-06,
+      "loss": 1.9602,
+      "step": 130
+    },
+    {
+      "epoch": 0.2721774193548387,
+      "grad_norm": 1.4165902883907349,
+      "learning_rate": 7.980341738939008e-06,
+      "loss": 1.985,
+      "step": 135
+    },
+    {
+      "epoch": 0.28225806451612906,
+      "grad_norm": 1.2768405135809222,
+      "learning_rate": 7.748230267366966e-06,
+      "loss": 1.9847,
+      "step": 140
+    },
+    {
+      "epoch": 0.2923387096774194,
+      "grad_norm": 1.4092721850037662,
+      "learning_rate": 7.522311335836008e-06,
+      "loss": 1.876,
+      "step": 145
+    },
+    {
+      "epoch": 0.3024193548387097,
+      "grad_norm": 1.307344104572624,
+      "learning_rate": 7.302435854911248e-06,
+      "loss": 1.9837,
+      "step": 150
+    },
+    {
+      "epoch": 0.3125,
+      "grad_norm": 1.1706462667165936,
+      "learning_rate": 7.088457934339014e-06,
+      "loss": 1.9666,
+      "step": 155
+    },
+    {
+      "epoch": 0.3225806451612903,
+      "grad_norm": 1.4451121840176004,
+      "learning_rate": 6.880234822817464e-06,
+      "loss": 1.9447,
+      "step": 160
+    },
+    {
+      "epoch": 0.3326612903225806,
+      "grad_norm": 1.5334691388320456,
+      "learning_rate": 6.677626848741738e-06,
+      "loss": 1.9565,
+      "step": 165
+    },
+    {
+      "epoch": 0.34274193548387094,
+      "grad_norm": 1.3638980480173026,
+      "learning_rate": 6.480497361910502e-06,
+      "loss": 1.8949,
+      "step": 170
+    },
+    {
+      "epoch": 0.3528225806451613,
+      "grad_norm": 1.1960476927370378,
+      "learning_rate": 6.288712676180814e-06,
+      "loss": 1.8283,
+      "step": 175
+    },
+    {
+      "epoch": 0.3629032258064516,
+      "grad_norm": 1.2534954529871944,
+      "learning_rate": 6.102142013058452e-06,
+      "loss": 1.9803,
+      "step": 180
+    },
+    {
+      "epoch": 0.37298387096774194,
+      "grad_norm": 1.371419907969094,
+      "learning_rate": 5.92065744621091e-06,
+      "loss": 1.966,
+      "step": 185
+    },
+    {
+      "epoch": 0.38306451612903225,
+      "grad_norm": 1.2633124617977696,
+      "learning_rate": 5.744133846890542e-06,
+      "loss": 1.9811,
+      "step": 190
+    },
+    {
+      "epoch": 0.39314516129032256,
+      "grad_norm": 1.164537124006197,
+      "learning_rate": 5.572448830255249e-06,
+      "loss": 1.9782,
+      "step": 195
+    },
+    {
+      "epoch": 0.4032258064516129,
+      "grad_norm": 1.2462948785459638,
+      "learning_rate": 5.405482702574484e-06,
+      "loss": 1.9087,
+      "step": 200
+    },
+    {
+      "epoch": 0.4032258064516129,
+      "eval_loss": 1.971670150756836,
+      "eval_runtime": 90.0126,
+      "eval_samples_per_second": 19.564,
+      "eval_steps_per_second": 0.311,
+      "step": 200
+    },
+    {
+      "epoch": 0.41330645161290325,
+      "grad_norm": 1.3766492549691904,
+      "learning_rate": 5.243118409308317e-06,
+      "loss": 2.0798,
+      "step": 205
+    },
+    {
+      "epoch": 0.42338709677419356,
+      "grad_norm": 1.2423997076500177,
+      "learning_rate": 5.085241484047513e-06,
+      "loss": 1.8739,
+      "step": 210
+    },
+    {
+      "epoch": 0.4334677419354839,
+      "grad_norm": 1.1300210393690464,
+      "learning_rate": 4.931739998302693e-06,
+      "loss": 1.8057,
+      "step": 215
+    },
+    {
+      "epoch": 0.4435483870967742,
+      "grad_norm": 1.4140153817527936,
+      "learning_rate": 4.782504512130776e-06,
+      "loss": 1.9858,
+      "step": 220
+    },
+    {
+      "epoch": 0.4536290322580645,
+      "grad_norm": 1.2444950990794887,
+      "learning_rate": 4.637428025587017e-06,
+      "loss": 1.9789,
+      "step": 225
+    },
+    {
+      "epoch": 0.4637096774193548,
+      "grad_norm": 1.2052674239904773,
+      "learning_rate": 4.4964059309911556e-06,
+      "loss": 1.9367,
+      "step": 230
+    },
+    {
+      "epoch": 0.4737903225806452,
+      "grad_norm": 1.277871648485073,
+      "learning_rate": 4.3593359659961716e-06,
+      "loss": 1.9616,
+      "step": 235
+    },
+    {
+      "epoch": 0.4838709677419355,
+      "grad_norm": 1.3535390602470023,
+      "learning_rate": 4.226118167448512e-06,
+      "loss": 1.9542,
+      "step": 240
+    },
+    {
+      "epoch": 0.4939516129032258,
+      "grad_norm": 1.222656117941944,
+      "learning_rate": 4.096654826028485e-06,
+      "loss": 1.9179,
+      "step": 245
+    },
+    {
+      "epoch": 0.5040322580645161,
+      "grad_norm": 1.2370551856793075,
+      "learning_rate": 3.97085044165989e-06,
+      "loss": 1.8558,
+      "step": 250
+    },
+    {
+      "epoch": 0.5141129032258065,
+      "grad_norm": 1.2889675743730276,
+      "learning_rate": 3.848611679677959e-06,
+      "loss": 1.9233,
+      "step": 255
+    },
+    {
+      "epoch": 0.5241935483870968,
+      "grad_norm": 1.2084336693966136,
+      "learning_rate": 3.729847327744829e-06,
+      "loss": 1.8753,
+      "step": 260
+    },
+    {
+      "epoch": 0.5342741935483871,
+      "grad_norm": 1.4153228238243372,
+      "learning_rate": 3.6144682535018863e-06,
+      "loss": 1.9414,
+      "step": 265
+    },
+    {
+      "epoch": 0.5443548387096774,
+      "grad_norm": 1.2415679119854826,
+      "learning_rate": 3.502387362948446e-06,
+      "loss": 1.9364,
+      "step": 270
+    },
+    {
+      "epoch": 0.5544354838709677,
+      "grad_norm": 1.2442629747270402,
+      "learning_rate": 3.3935195595364e-06,
+      "loss": 1.8724,
+      "step": 275
+    },
+    {
+      "epoch": 0.5645161290322581,
+      "grad_norm": 1.3555733119122653,
+      "learning_rate": 3.287781703970469e-06,
+      "loss": 1.8838,
+      "step": 280
+    },
+    {
+      "epoch": 0.5745967741935484,
+      "grad_norm": 1.2470604787323722,
+      "learning_rate": 3.1850925747039375e-06,
+      "loss": 1.994,
+      "step": 285
+    },
+    {
+      "epoch": 0.5846774193548387,
+      "grad_norm": 1.35644231599274,
+      "learning_rate": 3.085372829119804e-06,
+      "loss": 1.897,
+      "step": 290
+    },
+    {
+      "epoch": 0.594758064516129,
+      "grad_norm": 1.326596637954423,
+      "learning_rate": 2.9885449653873967e-06,
+      "loss": 1.8701,
+      "step": 295
+    },
+    {
+      "epoch": 0.6048387096774194,
+      "grad_norm": 1.1997391649395446,
+      "learning_rate": 2.8945332849846483e-06,
+      "loss": 1.9457,
+      "step": 300
+    },
+    {
+      "epoch": 0.6149193548387096,
+      "grad_norm": 1.25341747302999,
+      "learning_rate": 2.8032638558763024e-06,
+      "loss": 1.8957,
+      "step": 305
+    },
+    {
+      "epoch": 0.625,
+      "grad_norm": 1.3099633124688492,
+      "learning_rate": 2.714664476338476e-06,
+      "loss": 1.7966,
+      "step": 310
+    },
+    {
+      "epoch": 0.6350806451612904,
+      "grad_norm": 1.2383580405168713,
+      "learning_rate": 2.6286646394200784e-06,
+      "loss": 1.8531,
+      "step": 315
+    },
+    {
+      "epoch": 0.6451612903225806,
+      "grad_norm": 1.1594361027274587,
+      "learning_rate": 2.5451954980317313e-06,
+      "loss": 2.0083,
+      "step": 320
+    },
+    {
+      "epoch": 0.655241935483871,
+      "grad_norm": 1.2490365651731397,
+      "learning_rate": 2.4641898306529013e-06,
+      "loss": 2.0424,
+      "step": 325
+    },
+    {
+      "epoch": 0.6653225806451613,
+      "grad_norm": 1.181798792519064,
+      "learning_rate": 2.385582007648151e-06,
+      "loss": 1.927,
+      "step": 330
+    },
+    {
+      "epoch": 0.6754032258064516,
+      "grad_norm": 1.1464146712668952,
+      "learning_rate": 2.3093079581833982e-06,
+      "loss": 1.841,
+      "step": 335
+    },
+    {
+      "epoch": 0.6854838709677419,
+      "grad_norm": 1.272807320496496,
+      "learning_rate": 2.2353051377332933e-06,
+      "loss": 2.0111,
+      "step": 340
+    },
+    {
+      "epoch": 0.6955645161290323,
+      "grad_norm": 1.294506676739782,
+      "learning_rate": 2.163512496170876e-06,
+      "loss": 1.8891,
+      "step": 345
+    },
+    {
+      "epoch": 0.7056451612903226,
+      "grad_norm": 1.1893154239696047,
+      "learning_rate": 2.093870446430792e-06,
+      "loss": 1.9165,
+      "step": 350
+    },
+    {
+      "epoch": 0.7157258064516129,
+      "grad_norm": 1.3456137263979548,
+      "learning_rate": 2.026320833737434e-06,
+      "loss": 1.9668,
+      "step": 355
+    },
+    {
+      "epoch": 0.7258064516129032,
+      "grad_norm": 1.237670919084104,
+      "learning_rate": 1.9608069053895406e-06,
+      "loss": 1.8764,
+      "step": 360
+    },
+    {
+      "epoch": 0.7358870967741935,
+      "grad_norm": 1.2467674896007754,
+      "learning_rate": 1.8972732810927808e-06,
+      "loss": 1.8413,
+      "step": 365
+    },
+    {
+      "epoch": 0.7459677419354839,
+      "grad_norm": 1.2641168575246482,
+      "learning_rate": 1.8356659238320686e-06,
+      "loss": 1.8496,
+      "step": 370
+    },
+    {
+      "epoch": 0.7560483870967742,
+      "grad_norm": 1.3760663460506264,
+      "learning_rate": 1.7759321112753693e-06,
+      "loss": 1.8394,
+      "step": 375
+    },
+    {
+      "epoch": 0.7661290322580645,
+      "grad_norm": 1.1549697025197416,
+      "learning_rate": 1.718020407700906e-06,
+      "loss": 1.8772,
+      "step": 380
+    },
+    {
+      "epoch": 0.7762096774193549,
+      "grad_norm": 1.173935455479261,
+      "learning_rate": 1.6618806364397543e-06,
+      "loss": 1.873,
+      "step": 385
+    },
+    {
+      "epoch": 0.7862903225806451,
+      "grad_norm": 1.1687914205281313,
+      "learning_rate": 1.6074638528259137e-06,
+      "loss": 1.7951,
+      "step": 390
+    },
+    {
+      "epoch": 0.7963709677419355,
+      "grad_norm": 1.2122222438808594,
+      "learning_rate": 1.5547223176460478e-06,
+      "loss": 1.9303,
+      "step": 395
+    },
+    {
+      "epoch": 0.8064516129032258,
+      "grad_norm": 1.2659166954554237,
+      "learning_rate": 1.503609471081174e-06,
+      "loss": 1.8752,
+      "step": 400
+    },
+    {
+      "epoch": 0.8064516129032258,
+      "eval_loss": 1.9417628049850464,
+      "eval_runtime": 89.9499,
+      "eval_samples_per_second": 19.578,
+      "eval_steps_per_second": 0.311,
+      "step": 400
+    },
+    {
+      "epoch": 0.8165322580645161,
+      "grad_norm": 1.3170841848170978,
+      "learning_rate": 1.4540799071326888e-06,
+      "loss": 1.916,
+      "step": 405
+    },
+    {
+      "epoch": 0.8266129032258065,
+      "grad_norm": 1.2106133937481183,
+      "learning_rate": 1.4060893485251885e-06,
+      "loss": 1.9357,
+      "step": 410
+    },
+    {
+      "epoch": 0.8366935483870968,
+      "grad_norm": 1.1609644269427515,
+      "learning_rate": 1.3595946220786867e-06,
+      "loss": 1.9729,
+      "step": 415
+    },
+    {
+      "epoch": 0.8467741935483871,
+      "grad_norm": 1.255134988378285,
+      "learning_rate": 1.3145536345428467e-06,
+      "loss": 1.8483,
+      "step": 420
+    },
+    {
+      "epoch": 0.8568548387096774,
+      "grad_norm": 1.2271496366595076,
+      "learning_rate": 1.2709253488860179e-06,
+      "loss": 1.9611,
+      "step": 425
+    },
+    {
+      "epoch": 0.8669354838709677,
+      "grad_norm": 1.2112912625332515,
+      "learning_rate": 1.228669761031903e-06,
+      "loss": 1.8703,
+      "step": 430
+    },
+    {
+      "epoch": 0.8770161290322581,
+      "grad_norm": 1.1796106189216746,
+      "learning_rate": 1.1877478770367983e-06,
+      "loss": 1.8931,
+      "step": 435
+    },
+    {
+      "epoch": 0.8870967741935484,
+      "grad_norm": 1.2743988647789708,
+      "learning_rate": 1.1481216907004316e-06,
+      "loss": 2.0008,
+      "step": 440
+    },
+    {
+      "epoch": 0.8971774193548387,
+      "grad_norm": 1.109580598205878,
+      "learning_rate": 1.109754161603501e-06,
+      "loss": 1.8754,
+      "step": 445
+    },
+    {
+      "epoch": 0.907258064516129,
+      "grad_norm": 1.3265747651562068,
+      "learning_rate": 1.07260919356514e-06,
+      "loss": 1.937,
+      "step": 450
+    },
+    {
+      "epoch": 0.9173387096774194,
+      "grad_norm": 1.2255846935811405,
+      "learning_rate": 1.036651613513567e-06,
+      "loss": 1.839,
+      "step": 455
+    },
+    {
+      "epoch": 0.9274193548387096,
+      "grad_norm": 1.1637588001750867,
+      "learning_rate": 1.0018471507633137e-06,
+      "loss": 1.8875,
+      "step": 460
+    },
+    {
+      "epoch": 0.9375,
+      "grad_norm": 1.2461604557758548,
+      "learning_rate": 9.681624166924873e-07,
+      "loss": 2.0349,
+      "step": 465
+    },
+    {
+      "epoch": 0.9475806451612904,
+      "grad_norm": 1.106959630092776,
+      "learning_rate": 9.355648848136057e-07,
+      "loss": 1.7721,
+      "step": 470
+    },
+    {
+      "epoch": 0.9576612903225806,
+      "grad_norm": 1.1845513909345546,
+      "learning_rate": 9.040228712316404e-07,
+      "loss": 1.9764,
+      "step": 475
+    },
+    {
+      "epoch": 0.967741935483871,
+      "grad_norm": 1.1157403262103818,
+      "learning_rate": 8.735055154829718e-07,
+      "loss": 1.951,
+      "step": 480
+    },
+    {
+      "epoch": 0.9778225806451613,
+      "grad_norm": 1.184174473692059,
+      "learning_rate": 8.439827617490521e-07,
+      "loss": 1.9679,
+      "step": 485
+    },
+    {
+      "epoch": 0.9879032258064516,
+      "grad_norm": 1.2877898930931635,
+      "learning_rate": 8.154253404386484e-07,
+      "loss": 1.9346,
+      "step": 490
+    },
+    {
+      "epoch": 0.9979838709677419,
+      "grad_norm": 1.1827480417909397,
+      "learning_rate": 7.878047501326227e-07,
+      "loss": 1.9182,
+      "step": 495
+    },
+    {
+      "epoch": 1.0080645161290323,
+      "grad_norm": 1.3196758723195976,
+      "learning_rate": 7.61093239885272e-07,
+      "loss": 1.7384,
+      "step": 500
+    },
+    {
+      "epoch": 1.0181451612903225,
+      "grad_norm": 1.2675698548410592,
+      "learning_rate": 7.35263791876364e-07,
+      "loss": 1.7799,
+      "step": 505
+    },
+    {
+      "epoch": 1.028225806451613,
+      "grad_norm": 1.3275129553040272,
+      "learning_rate": 7.102901044080282e-07,
+      "loss": 1.7003,
+      "step": 510
+    },
+    {
+      "epoch": 1.0383064516129032,
+      "grad_norm": 1.4098711468145062,
+      "learning_rate": 6.861465752407945e-07,
+      "loss": 1.7438,
+      "step": 515
+    },
+    {
+      "epoch": 1.0483870967741935,
+      "grad_norm": 1.46840354164103,
+      "learning_rate": 6.628082852631131e-07,
+      "loss": 1.8117,
+      "step": 520
+    },
+    {
+      "epoch": 1.0584677419354838,
+      "grad_norm": 1.2413756391084652,
+      "learning_rate": 6.402509824887728e-07,
+      "loss": 1.7275,
+      "step": 525
+    },
+    {
+      "epoch": 1.0685483870967742,
+      "grad_norm": 1.1966512738085646,
+      "learning_rate": 6.184510663767266e-07,
+      "loss": 1.7607,
+      "step": 530
+    },
+    {
+      "epoch": 1.0786290322580645,
+      "grad_norm": 1.1871529707057311,
+      "learning_rate": 5.973855724678734e-07,
+      "loss": 1.7508,
+      "step": 535
+    },
+    {
+      "epoch": 1.0887096774193548,
+      "grad_norm": 1.2178547151060322,
+      "learning_rate": 5.770321573334551e-07,
+      "loss": 1.7386,
+      "step": 540
+    },
+    {
+      "epoch": 1.0987903225806452,
+      "grad_norm": 1.215474948218683,
+      "learning_rate": 5.573690838297752e-07,
+      "loss": 1.7264,
+      "step": 545
+    },
+    {
+      "epoch": 1.1088709677419355,
+      "grad_norm": 1.3022255779670897,
+      "learning_rate": 5.383752066540304e-07,
+      "loss": 1.7367,
+      "step": 550
+    },
+    {
+      "epoch": 1.1189516129032258,
+      "grad_norm": 1.2677022586640083,
+      "learning_rate": 5.200299581961105e-07,
+      "loss": 1.8039,
+      "step": 555
+    },
+    {
+      "epoch": 1.129032258064516,
+      "grad_norm": 1.2498635415826649,
+      "learning_rate": 5.023133346812977e-07,
+      "loss": 1.7811,
+      "step": 560
+    },
+    {
+      "epoch": 1.1391129032258065,
+      "grad_norm": 1.1813135414245424,
+      "learning_rate": 4.85205882598862e-07,
+      "loss": 1.7935,
+      "step": 565
+    },
+    {
+      "epoch": 1.1491935483870968,
+      "grad_norm": 1.283761351053989,
+      "learning_rate": 4.686886854116219e-07,
+      "loss": 1.7428,
+      "step": 570
+    },
+    {
+      "epoch": 1.159274193548387,
+      "grad_norm": 1.181148159521782,
+      "learning_rate": 4.5274335054160525e-07,
+      "loss": 1.7405,
+      "step": 575
+    },
+    {
+      "epoch": 1.1693548387096775,
+      "grad_norm": 1.2540563610260587,
+      "learning_rate": 4.3735199662701383e-07,
+      "loss": 1.7017,
+      "step": 580
+    },
+    {
+      "epoch": 1.1794354838709677,
+      "grad_norm": 1.26673278964677,
+      "learning_rate": 4.2249724104576337e-07,
+      "loss": 1.7258,
+      "step": 585
+    },
+    {
+      "epoch": 1.189516129032258,
+      "grad_norm": 1.2483740436576478,
+      "learning_rate": 4.081621877009305e-07,
+      "loss": 1.7842,
+      "step": 590
+    },
+    {
+      "epoch": 1.1995967741935485,
+      "grad_norm": 1.2338327909706366,
+      "learning_rate": 3.9433041506352036e-07,
+      "loss": 1.7602,
+      "step": 595
+    },
+    {
+      "epoch": 1.2096774193548387,
+      "grad_norm": 1.2070627326429186,
+      "learning_rate": 3.8098596446800587e-07,
+      "loss": 1.6383,
+      "step": 600
+    },
+    {
+      "epoch": 1.2096774193548387,
+      "eval_loss": 1.9439929723739624,
+      "eval_runtime": 89.7597,
+      "eval_samples_per_second": 19.619,
+      "eval_steps_per_second": 0.312,
+      "step": 600
+    },
+    {
+      "epoch": 1.219758064516129,
+      "grad_norm": 1.2154229290189673,
+      "learning_rate": 3.681133286561828e-07,
+      "loss": 1.8134,
+      "step": 605
+    },
+    {
+      "epoch": 1.2298387096774193,
+      "grad_norm": 1.2295658806251533,
+      "learning_rate": 3.556974405649289e-07,
+      "loss": 1.725,
+      "step": 610
+    },
+    {
+      "epoch": 1.2399193548387097,
+      "grad_norm": 1.2791124588407303,
+      "learning_rate": 3.437236623535232e-07,
+      "loss": 1.7526,
+      "step": 615
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 1.1899209284700905,
+      "learning_rate": 3.3217777466625205e-07,
+      "loss": 1.7313,
+      "step": 620
+    },
+    {
+      "epoch": 1.2600806451612903,
+      "grad_norm": 1.3442383601969903,
+      "learning_rate": 3.2104596612607314e-07,
+      "loss": 1.8358,
+      "step": 625
+    },
+    {
+      "epoch": 1.2701612903225805,
+      "grad_norm": 1.2622913229084678,
+      "learning_rate": 3.1031482305518425e-07,
+      "loss": 1.7504,
+      "step": 630
+    },
+    {
+      "epoch": 1.280241935483871,
+      "grad_norm": 1.1574713957890075,
+      "learning_rate": 2.9997131941839894e-07,
+      "loss": 1.7646,
+      "step": 635
+    },
+    {
+      "epoch": 1.2903225806451613,
+      "grad_norm": 1.346661773742281,
+      "learning_rate": 2.9000280698528666e-07,
+      "loss": 1.7116,
+      "step": 640
+    },
+    {
+      "epoch": 1.3004032258064515,
+      "grad_norm": 1.3332483111343922,
+      "learning_rate": 2.8039700570709997e-07,
+      "loss": 1.8109,
+      "step": 645
+    },
+    {
+      "epoch": 1.310483870967742,
+      "grad_norm": 1.2333462778908426,
+      "learning_rate": 2.7114199430456553e-07,
+      "loss": 1.7683,
+      "step": 650
+    },
+    {
+      "epoch": 1.3205645161290323,
+      "grad_norm": 1.1938385640085034,
+      "learning_rate": 2.6222620106267406e-07,
+      "loss": 1.6945,
+      "step": 655
+    },
+    {
+      "epoch": 1.3306451612903225,
+      "grad_norm": 1.4010824213914208,
+      "learning_rate": 2.536383948286623e-07,
+      "loss": 1.7537,
+      "step": 660
+    },
+    {
+      "epoch": 1.340725806451613,
+      "grad_norm": 1.3881248076295551,
+      "learning_rate": 2.4536767620943495e-07,
+      "loss": 1.7843,
+      "step": 665
+    },
+    {
+      "epoch": 1.3508064516129032,
+      "grad_norm": 1.2168315394832683,
+      "learning_rate": 2.3740346896473077e-07,
+      "loss": 1.7928,
+      "step": 670
+    },
+    {
+      "epoch": 1.3608870967741935,
+      "grad_norm": 1.4475784692183353,
+      "learning_rate": 2.2973551159239254e-07,
+      "loss": 1.7755,
+      "step": 675
+    },
+    {
+      "epoch": 1.370967741935484,
+      "grad_norm": 1.182910838845923,
+      "learning_rate": 2.223538491021522e-07,
+      "loss": 1.7251,
+      "step": 680
+    },
+    {
+      "epoch": 1.3810483870967742,
+      "grad_norm": 1.226170490302768,
+      "learning_rate": 2.1524882497440346e-07,
+      "loss": 1.7814,
+      "step": 685
+    },
+    {
+      "epoch": 1.3911290322580645,
+      "grad_norm": 1.1753601198052888,
+      "learning_rate": 2.084110733004757e-07,
+      "loss": 1.7355,
+      "step": 690
+    },
+    {
+      "epoch": 1.4012096774193548,
+      "grad_norm": 1.176444951923013,
+      "learning_rate": 2.0183151110098748e-07,
+      "loss": 1.7006,
+      "step": 695
+    },
+    {
+      "epoch": 1.4112903225806452,
+      "grad_norm": 1.2575906929525473,
+      "learning_rate": 1.955013308189006e-07,
+      "loss": 1.6631,
+      "step": 700
+    },
+    {
+      "epoch": 1.4213709677419355,
+      "grad_norm": 1.3222267696078929,
+      "learning_rate": 1.8941199298395633e-07,
+      "loss": 1.7313,
+      "step": 705
+    },
+    {
+      "epoch": 1.4314516129032258,
+      "grad_norm": 1.3554981903288013,
+      "learning_rate": 1.8355521904521423e-07,
+      "loss": 1.7183,
+      "step": 710
+    },
+    {
+      "epoch": 1.441532258064516,
+      "grad_norm": 1.2333423736099849,
+      "learning_rate": 1.7792298436847624e-07,
+      "loss": 1.7057,
+      "step": 715
+    },
+    {
+      "epoch": 1.4516129032258065,
+      "grad_norm": 1.2791350480965469,
+      "learning_rate": 1.7250751139542018e-07,
+      "loss": 1.6957,
+      "step": 720
+    },
+    {
+      "epoch": 1.4616935483870968,
+      "grad_norm": 1.5651823236688753,
+      "learning_rate": 1.6730126296131982e-07,
+      "loss": 1.808,
+      "step": 725
+    },
+    {
+      "epoch": 1.471774193548387,
+      "grad_norm": 1.3083961982147532,
+      "learning_rate": 1.622969357682752e-07,
+      "loss": 1.7913,
+      "step": 730
+    },
+    {
+      "epoch": 1.4818548387096775,
+      "grad_norm": 1.2078318183364274,
+      "learning_rate": 1.574874540109262e-07,
+      "loss": 1.7273,
+      "step": 735
+    },
+    {
+      "epoch": 1.4919354838709677,
+      "grad_norm": 1.2904657184612536,
+      "learning_rate": 1.5286596315166833e-07,
+      "loss": 1.6646,
+      "step": 740
+    },
+    {
+      "epoch": 1.502016129032258,
+      "grad_norm": 1.240861344603422,
+      "learning_rate": 1.4842582384243795e-07,
+      "loss": 1.6495,
+      "step": 745
+    },
+    {
+      "epoch": 1.5120967741935485,
+      "grad_norm": 1.2574003293432507,
+      "learning_rate": 1.4416060599017888e-07,
+      "loss": 1.7356,
+      "step": 750
+    },
+    {
+      "epoch": 1.5221774193548387,
+      "grad_norm": 1.4067959486898025,
+      "learning_rate": 1.4006408296314945e-07,
+      "loss": 1.7802,
+      "step": 755
+    },
+    {
+      "epoch": 1.532258064516129,
+      "grad_norm": 1.3011200023092266,
+      "learning_rate": 1.3613022593527299e-07,
+      "loss": 1.7435,
+      "step": 760
+    },
+    {
+      "epoch": 1.5423387096774195,
+      "grad_norm": 1.5064955060135545,
+      "learning_rate": 1.3235319836577956e-07,
+      "loss": 1.8153,
+      "step": 765
+    },
+    {
+      "epoch": 1.5524193548387095,
+      "grad_norm": 1.2545915727253294,
+      "learning_rate": 1.287273506114331e-07,
+      "loss": 1.7333,
+      "step": 770
+    },
+    {
+      "epoch": 1.5625,
+      "grad_norm": 1.210516142345582,
+      "learning_rate": 1.2524721466867644e-07,
+      "loss": 1.8145,
+      "step": 775
+    },
+    {
+      "epoch": 1.5725806451612905,
+      "grad_norm": 1.2428363984374335,
+      "learning_rate": 1.2190749904307564e-07,
+      "loss": 1.7466,
+      "step": 780
+    },
+    {
+      "epoch": 1.5826612903225805,
+      "grad_norm": 1.2372984868597403,
+      "learning_rate": 1.187030837434837e-07,
+      "loss": 1.7707,
+      "step": 785
+    },
+    {
+      "epoch": 1.592741935483871,
+      "grad_norm": 1.223197065220676,
+      "learning_rate": 1.1562901539838627e-07,
+      "loss": 1.7515,
+      "step": 790
+    },
+    {
+      "epoch": 1.6028225806451613,
+      "grad_norm": 1.360461478384606,
+      "learning_rate": 1.1268050249193624e-07,
+      "loss": 1.7033,
+      "step": 795
+    },
+    {
+      "epoch": 1.6129032258064515,
+      "grad_norm": 1.3523444473608008,
+      "learning_rate": 1.0985291071721938e-07,
+      "loss": 1.7073,
+      "step": 800
+    },
+    {
+      "epoch": 1.6129032258064515,
+      "eval_loss": 1.9434727430343628,
+      "eval_runtime": 89.696,
+      "eval_samples_per_second": 19.633,
+      "eval_steps_per_second": 0.312,
+      "step": 800
+    },
+    {
+      "epoch": 1.622983870967742,
+      "grad_norm": 1.1908153349621502,
+      "learning_rate": 1.0714175844433986e-07,
+      "loss": 1.7606,
+      "step": 805
+    },
+    {
+      "epoch": 1.6330645161290323,
+      "grad_norm": 1.1559466036906028,
+      "learning_rate": 1.0454271230095008e-07,
+      "loss": 1.6855,
+      "step": 810
+    },
+    {
+      "epoch": 1.6431451612903225,
+      "grad_norm": 1.2177765725388214,
+      "learning_rate": 1.0205158286289079e-07,
+      "loss": 1.6938,
+      "step": 815
+    },
+    {
+      "epoch": 1.653225806451613,
+      "grad_norm": 1.2381331316689286,
+      "learning_rate": 9.966432045264543e-08,
+      "loss": 1.7047,
+      "step": 820
+    },
+    {
+      "epoch": 1.6633064516129032,
+      "grad_norm": 1.2662439850688683,
+      "learning_rate": 9.73770110433516e-08,
+      "loss": 1.8619,
+      "step": 825
+    },
+    {
+      "epoch": 1.6733870967741935,
+      "grad_norm": 1.3616392797318724,
+      "learning_rate": 9.518587226615035e-08,
+      "loss": 1.8544,
+      "step": 830
+    },
+    {
+      "epoch": 1.683467741935484,
+      "grad_norm": 1.2521663767495246,
+      "learning_rate": 9.30872495186915e-08,
+      "loss": 1.7568,
+      "step": 835
+    },
+    {
+      "epoch": 1.6935483870967742,
+      "grad_norm": 1.2504361983829193,
+      "learning_rate": 9.107761217265013e-08,
+      "loss": 1.721,
+      "step": 840
+    },
+    {
+      "epoch": 1.7036290322580645,
+      "grad_norm": 1.2617228722420664,
+      "learning_rate": 8.915354987814596e-08,
+      "loss": 1.7788,
+      "step": 845
+    },
+    {
+      "epoch": 1.713709677419355,
+      "grad_norm": 1.2060296867893439,
+      "learning_rate": 8.731176896299368e-08,
+      "loss": 1.7307,
+      "step": 850
+    },
+    {
+      "epoch": 1.723790322580645,
+      "grad_norm": 1.2261412385974246,
+      "learning_rate": 8.554908892474695e-08,
+      "loss": 1.6492,
+      "step": 855
+    },
+    {
+      "epoch": 1.7338709677419355,
+      "grad_norm": 1.1985063418448514,
+      "learning_rate": 8.38624390135359e-08,
+      "loss": 1.8062,
+      "step": 860
+    },
+    {
+      "epoch": 1.7439516129032258,
+      "grad_norm": 1.1687433855869847,
+      "learning_rate": 8.224885490372989e-08,
+      "loss": 1.6903,
+      "step": 865
+    },
+    {
+      "epoch": 1.754032258064516,
+      "grad_norm": 1.2094813386232657,
+      "learning_rate": 8.070547545249396e-08,
+      "loss": 1.7466,
+      "step": 870
+    },
+    {
+      "epoch": 1.7641129032258065,
+      "grad_norm": 1.5715777085972131,
+      "learning_rate": 7.922953954333971e-08,
+      "loss": 1.7321,
+      "step": 875
+    },
+    {
+      "epoch": 1.7741935483870968,
+      "grad_norm": 1.232854137306794,
+      "learning_rate": 7.781838301280604e-08,
+      "loss": 1.7854,
+      "step": 880
+    },
+    {
+      "epoch": 1.784274193548387,
+      "grad_norm": 1.2306505361743405,
+      "learning_rate": 7.646943565843591e-08,
+      "loss": 1.7035,
+      "step": 885
+    },
+    {
+      "epoch": 1.7943548387096775,
+      "grad_norm": 1.4038527293019962,
+      "learning_rate": 7.518021832625021e-08,
+      "loss": 1.7898,
+      "step": 890
+    },
+    {
+      "epoch": 1.8044354838709677,
+      "grad_norm": 1.1701930640429434,
+      "learning_rate": 7.39483400759502e-08,
+      "loss": 1.7473,
+      "step": 895
+    },
+    {
+      "epoch": 1.814516129032258,
+      "grad_norm": 1.165650133692535,
+      "learning_rate": 7.277149542211219e-08,
+      "loss": 1.7392,
+      "step": 900
+    },
+    {
+      "epoch": 1.8245967741935485,
+      "grad_norm": 1.242160120226514,
+      "learning_rate": 7.164746164966925e-08,
+      "loss": 1.691,
+      "step": 905
+    },
+    {
+      "epoch": 1.8346774193548387,
+      "grad_norm": 1.3662669550498554,
+      "learning_rate": 7.057409620200535e-08,
+      "loss": 1.7456,
+      "step": 910
+    },
+    {
+      "epoch": 1.844758064516129,
+      "grad_norm": 1.2736994338683274,
+      "learning_rate": 6.954933414001788e-08,
+      "loss": 1.9068,
+      "step": 915
+    },
+    {
+      "epoch": 1.8548387096774195,
+      "grad_norm": 1.3755517276213882,
+      "learning_rate": 6.857118567053417e-08,
+      "loss": 1.7568,
+      "step": 920
+    },
+    {
+      "epoch": 1.8649193548387095,
+      "grad_norm": 1.1731257131372528,
+      "learning_rate": 6.763773374249751e-08,
+      "loss": 1.7815,
+      "step": 925
+    },
+    {
+      "epoch": 1.875,
+      "grad_norm": 1.2300272384037476,
+      "learning_rate": 6.674713170936664e-08,
+      "loss": 1.7488,
+      "step": 930
+    },
+    {
+      "epoch": 1.8850806451612905,
+      "grad_norm": 1.2420985179509971,
+      "learning_rate": 6.589760105620217e-08,
+      "loss": 1.7614,
+      "step": 935
+    },
+    {
+      "epoch": 1.8951612903225805,
+      "grad_norm": 1.7224910659058157,
+      "learning_rate": 6.508742918994098e-08,
+      "loss": 1.7496,
+      "step": 940
+    },
+    {
+      "epoch": 1.905241935483871,
+      "grad_norm": 1.3013159637595342,
+      "learning_rate": 6.431496729138832e-08,
+      "loss": 1.7375,
+      "step": 945
+    },
+    {
+      "epoch": 1.9153225806451613,
+      "grad_norm": 1.2702415604664459,
+      "learning_rate": 6.357862822748362e-08,
+      "loss": 1.7668,
+      "step": 950
+    },
+    {
+      "epoch": 1.9254032258064515,
+      "grad_norm": 1.168954401665018,
+      "learning_rate": 6.287688452242478e-08,
+      "loss": 1.7489,
+      "step": 955
+    },
+    {
+      "epoch": 1.935483870967742,
+      "grad_norm": 1.3375224905639664,
+      "learning_rate": 6.220826638626108e-08,
+      "loss": 1.6308,
+      "step": 960
+    },
+    {
+      "epoch": 1.9455645161290323,
+      "grad_norm": 1.2577572377495214,
+      "learning_rate": 6.15713597995916e-08,
+      "loss": 1.7912,
+      "step": 965
+    },
+    {
+      "epoch": 1.9556451612903225,
+      "grad_norm": 1.2336415332120372,
+      "learning_rate": 6.096480465303296e-08,
+      "loss": 1.7562,
+      "step": 970
+    },
+    {
+      "epoch": 1.965725806451613,
+      "grad_norm": 1.3102010598064548,
+      "learning_rate": 6.038729294014376e-08,
+      "loss": 1.6953,
+      "step": 975
+    },
+    {
+      "epoch": 1.9758064516129032,
+      "grad_norm": 1.3132413812361838,
+      "learning_rate": 5.983756700252073e-08,
+      "loss": 1.7719,
+      "step": 980
+    },
+    {
+      "epoch": 1.9858870967741935,
+      "grad_norm": 1.2310128233562452,
+      "learning_rate": 5.9314417825804556e-08,
+      "loss": 1.7212,
+      "step": 985
+    },
+    {
+      "epoch": 1.995967741935484,
+      "grad_norm": 1.2006191684102008,
+      "learning_rate": 5.8816683385358974e-08,
+      "loss": 1.7667,
+      "step": 990
+    },
+    {
+      "epoch": 2.006048387096774,
+      "grad_norm": 1.2035537415937343,
+      "learning_rate": 5.834324704041041e-08,
+      "loss": 1.7686,
+      "step": 995
+    },
+    {
+      "epoch": 2.0161290322580645,
+      "grad_norm": 1.2316714422775488,
+      "learning_rate": 5.789303597545942e-08,
+      "loss": 1.6699,
+      "step": 1000
+    },
+    {
+      "epoch": 2.0161290322580645,
+      "eval_loss": 1.9428125619888306,
+      "eval_runtime": 89.8315,
+      "eval_samples_per_second": 19.603,
+      "eval_steps_per_second": 0.312,
+      "step": 1000
+    },
+    {
+      "epoch": 2.026209677419355,
+      "grad_norm": 1.2569040983124893,
+      "learning_rate": 5.746501968779849e-08,
+      "loss": 1.7248,
+      "step": 1005
+    },
+    {
+      "epoch": 2.036290322580645,
+      "grad_norm": 1.2073156251056785,
+      "learning_rate": 5.7058208519993964e-08,
+      "loss": 1.5994,
+      "step": 1010
+    },
+    {
+      "epoch": 2.0463709677419355,
+      "grad_norm": 1.250476166765303,
+      "learning_rate": 5.667165223621266e-08,
+      "loss": 1.7618,
+      "step": 1015
+    },
+    {
+      "epoch": 2.056451612903226,
+      "grad_norm": 1.1771740320614876,
+      "learning_rate": 5.630443864129586e-08,
+      "loss": 1.7109,
+      "step": 1020
+    },
+    {
+      "epoch": 2.066532258064516,
+      "grad_norm": 1.2423874053675774,
+      "learning_rate": 5.595569224150602e-08,
+      "loss": 1.688,
+      "step": 1025
+    },
+    {
+      "epoch": 2.0766129032258065,
+      "grad_norm": 1.1913045210590958,
+      "learning_rate": 5.562457294589246e-08,
+      "loss": 1.7122,
+      "step": 1030
+    },
+    {
+      "epoch": 2.086693548387097,
+      "grad_norm": 1.5628319901302377,
+      "learning_rate": 5.53102748072445e-08,
+      "loss": 1.8032,
+      "step": 1035
+    },
+    {
+      "epoch": 2.096774193548387,
+      "grad_norm": 1.2186459343954958,
+      "learning_rate": 5.501202480162106e-08,
+      "loss": 1.706,
+      "step": 1040
+    },
+    {
+      "epoch": 2.1068548387096775,
+      "grad_norm": 1.5980051096787016,
+      "learning_rate": 5.472908164546672e-08,
+      "loss": 1.6859,
+      "step": 1045
+    },
+    {
+      "epoch": 2.1169354838709675,
+      "grad_norm": 1.2032746720272727,
+      "learning_rate": 5.446073464934468e-08,
+      "loss": 1.6677,
+      "step": 1050
+    },
+    {
+      "epoch": 2.127016129032258,
+      "grad_norm": 1.2566967045401873,
+      "learning_rate": 5.420630260733693e-08,
+      "loss": 1.7704,
+      "step": 1055
+    },
+    {
+      "epoch": 2.1370967741935485,
+      "grad_norm": 1.1883212669149759,
+      "learning_rate": 5.396513272118224e-08,
+      "loss": 1.7207,
+      "step": 1060
+    },
+    {
+      "epoch": 2.1471774193548385,
+      "grad_norm": 1.4452669028801322,
+      "learning_rate": 5.373659955824137e-08,
+      "loss": 1.7838,
+      "step": 1065
+    },
+    {
+      "epoch": 2.157258064516129,
+      "grad_norm": 1.2860799707706096,
+      "learning_rate": 5.3520104042398855e-08,
+      "loss": 1.7416,
+      "step": 1070
+    },
+    {
+      "epoch": 2.1673387096774195,
+      "grad_norm": 1.1967308129229457,
+      "learning_rate": 5.331507247702889e-08,
+      "loss": 1.6755,
+      "step": 1075
+    },
+    {
+      "epoch": 2.1774193548387095,
+      "grad_norm": 1.2537902733229633,
+      "learning_rate": 5.3120955599171935e-08,
+      "loss": 1.7548,
+      "step": 1080
+    },
+    {
+      "epoch": 2.1875,
+      "grad_norm": 1.2447529415534258,
+      "learning_rate": 5.293722766408659e-08,
+      "loss": 1.7471,
+      "step": 1085
+    },
+    {
+      "epoch": 2.1975806451612905,
+      "grad_norm": 1.1991864633094496,
+      "learning_rate": 5.276338555935963e-08,
+      "loss": 1.7606,
+      "step": 1090
+    },
+    {
+      "epoch": 2.2076612903225805,
+      "grad_norm": 1.219389195740527,
+      "learning_rate": 5.259894794777423e-08,
+      "loss": 1.7225,
+      "step": 1095
+    },
+    {
+      "epoch": 2.217741935483871,
+      "grad_norm": 1.2339738380016207,
+      "learning_rate": 5.2443454438154534e-08,
+      "loss": 1.7114,
+      "step": 1100
+    },
+    {
+      "epoch": 2.2278225806451615,
+      "grad_norm": 1.1802337869081512,
+      "learning_rate": 5.229646478342111e-08,
+      "loss": 1.694,
+      "step": 1105
+    },
+    {
+      "epoch": 2.2379032258064515,
+      "grad_norm": 1.2158683240460215,
+      "learning_rate": 5.2157558105109226e-08,
+      "loss": 1.7462,
+      "step": 1110
+    },
+    {
+      "epoch": 2.247983870967742,
+      "grad_norm": 1.174384875714619,
+      "learning_rate": 5.202633214361811e-08,
+      "loss": 1.695,
+      "step": 1115
+    },
+    {
+      "epoch": 2.258064516129032,
+      "grad_norm": 1.2247069091252694,
+      "learning_rate": 5.190240253347578e-08,
+      "loss": 1.7591,
+      "step": 1120
+    },
+    {
+      "epoch": 2.2681451612903225,
+      "grad_norm": 1.3095787384406332,
+      "learning_rate": 5.178540210291998e-08,
+      "loss": 1.7217,
+      "step": 1125
+    },
+    {
+      "epoch": 2.278225806451613,
+      "grad_norm": 1.2493489539759923,
+      "learning_rate": 5.167498019711146e-08,
+      "loss": 1.7889,
+      "step": 1130
+    },
+    {
+      "epoch": 2.288306451612903,
+      "grad_norm": 1.2374059562548885,
+      "learning_rate": 5.157080202431146e-08,
+      "loss": 1.6853,
+      "step": 1135
+    },
+    {
+      "epoch": 2.2983870967741935,
+      "grad_norm": 1.2417228451011928,
+      "learning_rate": 5.1472548024370265e-08,
+      "loss": 1.7252,
+      "step": 1140
+    },
+    {
+      "epoch": 2.308467741935484,
+      "grad_norm": 1.2382899521975939,
+      "learning_rate": 5.137991325888869e-08,
+      "loss": 1.6828,
+      "step": 1145
+    },
+    {
+      "epoch": 2.318548387096774,
+      "grad_norm": 1.2406554761874888,
+      "learning_rate": 5.129260682242921e-08,
+      "loss": 1.7995,
+      "step": 1150
+    },
+    {
+      "epoch": 2.3286290322580645,
+      "grad_norm": 1.2857931147752715,
+      "learning_rate": 5.121035127416763e-08,
+      "loss": 1.6909,
+      "step": 1155
+    },
+    {
+      "epoch": 2.338709677419355,
+      "grad_norm": 1.2227137154473942,
+      "learning_rate": 5.1132882089390505e-08,
+      "loss": 1.8297,
+      "step": 1160
+    },
+    {
+      "epoch": 2.348790322580645,
+      "grad_norm": 1.2092856119101558,
+      "learning_rate": 5.105994713025749e-08,
+      "loss": 1.7084,
+      "step": 1165
+    },
+    {
+      "epoch": 2.3588709677419355,
+      "grad_norm": 1.2553731796947716,
+      "learning_rate": 5.099130613526128e-08,
+      "loss": 1.7698,
+      "step": 1170
+    },
+    {
+      "epoch": 2.368951612903226,
+      "grad_norm": 1.2347295734439494,
+      "learning_rate": 5.092673022683157e-08,
+      "loss": 1.7176,
+      "step": 1175
+    },
+    {
+      "epoch": 2.379032258064516,
+      "grad_norm": 1.2236546996683366,
+      "learning_rate": 5.086600143654227e-08,
+      "loss": 1.7972,
+      "step": 1180
+    },
+    {
+      "epoch": 2.3891129032258065,
+      "grad_norm": 1.1892980030528701,
+      "learning_rate": 5.080891224739457e-08,
+      "loss": 1.7075,
+      "step": 1185
+    },
+    {
+      "epoch": 2.399193548387097,
+      "grad_norm": 1.2693570071261813,
+      "learning_rate": 5.075526515266076e-08,
+      "loss": 1.762,
+      "step": 1190
+    },
+    {
+      "epoch": 2.409274193548387,
+      "grad_norm": 1.2598507176307956,
+      "learning_rate": 5.07048722307867e-08,
+      "loss": 1.7596,
+      "step": 1195
+    },
+    {
+      "epoch": 2.4193548387096775,
+      "grad_norm": 1.2086250967210856,
+      "learning_rate": 5.065755473586252e-08,
+      "loss": 1.7212,
+      "step": 1200
+    },
+    {
+      "epoch": 2.4193548387096775,
+      "eval_loss": 1.9444948434829712,
+      "eval_runtime": 89.768,
+      "eval_samples_per_second": 19.617,
+      "eval_steps_per_second": 0.312,
+      "step": 1200
+    },
+    {
+      "epoch": 2.429435483870968,
+      "grad_norm": 1.2270017834004545,
+      "learning_rate": 5.061314270318372e-08,
+      "loss": 1.7673,
+      "step": 1205
+    },
+    {
+      "epoch": 2.439516129032258,
+      "grad_norm": 1.2758334557746702,
+      "learning_rate": 5.057147456943636e-08,
+      "loss": 1.7733,
+      "step": 1210
+    },
+    {
+      "epoch": 2.4495967741935485,
+      "grad_norm": 1.2671506994834478,
+      "learning_rate": 5.053239680705157e-08,
+      "loss": 1.685,
+      "step": 1215
+    },
+    {
+      "epoch": 2.4596774193548385,
+      "grad_norm": 1.3082114423304811,
+      "learning_rate": 5.049576357228625e-08,
+      "loss": 1.7673,
+      "step": 1220
+    },
+    {
+      "epoch": 2.469758064516129,
+      "grad_norm": 1.2410498924874347,
+      "learning_rate": 5.0461436366597737e-08,
+      "loss": 1.6777,
+      "step": 1225
+    },
+    {
+      "epoch": 2.4798387096774195,
+      "grad_norm": 1.2225089432500613,
+      "learning_rate": 5.0429283710891395e-08,
+      "loss": 1.7561,
+      "step": 1230
+    },
+    {
+      "epoch": 2.4899193548387095,
+      "grad_norm": 1.3411537670367075,
+      "learning_rate": 5.0399180832230535e-08,
+      "loss": 1.7278,
+      "step": 1235
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 1.2608235504861707,
+      "learning_rate": 5.0371009362608996e-08,
+      "loss": 1.6774,
+      "step": 1240
+    },
+    {
+      "epoch": 2.5100806451612905,
+      "grad_norm": 1.2549696696310126,
+      "learning_rate": 5.0344657049396644e-08,
+      "loss": 1.6711,
+      "step": 1245
+    },
+    {
+      "epoch": 2.5201612903225805,
+      "grad_norm": 1.2520913106199976,
+      "learning_rate": 5.032001747707849e-08,
+      "loss": 1.7873,
+      "step": 1250
+    },
+    {
+      "epoch": 2.530241935483871,
+      "grad_norm": 1.2195192727589492,
+      "learning_rate": 5.029698979991788e-08,
+      "loss": 1.7961,
+      "step": 1255
+    },
+    {
+      "epoch": 2.540322580645161,
+      "grad_norm": 1.2296378280476938,
+      "learning_rate": 5.0275478485184044e-08,
+      "loss": 1.7105,
+      "step": 1260
+    },
+    {
+      "epoch": 2.5504032258064515,
+      "grad_norm": 1.2672207613505957,
+      "learning_rate": 5.025539306659372e-08,
+      "loss": 1.7319,
+      "step": 1265
+    },
+    {
+      "epoch": 2.560483870967742,
+      "grad_norm": 1.33348278916816,
+      "learning_rate": 5.0236647907626076e-08,
+      "loss": 1.7581,
+      "step": 1270
+    },
+    {
+      "epoch": 2.570564516129032,
+      "grad_norm": 1.2147181184983937,
+      "learning_rate": 5.021916197437918e-08,
+      "loss": 1.7909,
+      "step": 1275
+    },
+    {
+      "epoch": 2.5806451612903225,
+      "grad_norm": 1.2006424516658476,
+      "learning_rate": 5.020285861764526e-08,
+      "loss": 1.7094,
+      "step": 1280
+    },
+    {
+      "epoch": 2.590725806451613,
+      "grad_norm": 1.304898262044129,
+      "learning_rate": 5.018766536389094e-08,
+      "loss": 1.7601,
+      "step": 1285
+    },
+    {
+      "epoch": 2.600806451612903,
+      "grad_norm": 1.2541227980327716,
+      "learning_rate": 5.017351371483695e-08,
+      "loss": 1.6886,
+      "step": 1290
+    },
+    {
+      "epoch": 2.6108870967741935,
+      "grad_norm": 1.2252553715485226,
+      "learning_rate": 5.01603389553406e-08,
+      "loss": 1.7265,
+      "step": 1295
+    },
+    {
+      "epoch": 2.620967741935484,
+      "grad_norm": 1.249302955897003,
+      "learning_rate": 5.0148079969292275e-08,
+      "loss": 1.6571,
+      "step": 1300
+    },
+    {
+      "epoch": 2.631048387096774,
+      "grad_norm": 1.1957123027809284,
+      "learning_rate": 5.0136679063245423e-08,
+      "loss": 1.7532,
+      "step": 1305
+    },
+    {
+      "epoch": 2.6411290322580645,
+      "grad_norm": 1.201048445138404,
+      "learning_rate": 5.012608179750745e-08,
+      "loss": 1.7326,
+      "step": 1310
+    },
+    {
+      "epoch": 2.651209677419355,
+      "grad_norm": 1.2081583082273468,
+      "learning_rate": 5.011623682442653e-08,
+      "loss": 1.6896,
+      "step": 1315
+    },
+    {
+      "epoch": 2.661290322580645,
+      "grad_norm": 1.2165377024812585,
+      "learning_rate": 5.010709573361717e-08,
+      "loss": 1.7138,
+      "step": 1320
+    },
+    {
+      "epoch": 2.6713709677419355,
+      "grad_norm": 1.2512113015598476,
+      "learning_rate": 5.0098612903874436e-08,
+      "loss": 1.7417,
+      "step": 1325
+    },
+    {
+      "epoch": 2.681451612903226,
+      "grad_norm": 1.3393843580639568,
+      "learning_rate": 5.009074536153439e-08,
+      "loss": 1.6794,
+      "step": 1330
+    },
+    {
+      "epoch": 2.691532258064516,
+      "grad_norm": 1.244775300561878,
+      "learning_rate": 5.008345264504506e-08,
+      "loss": 1.6785,
+      "step": 1335
+    },
+    {
+      "epoch": 2.7016129032258065,
+      "grad_norm": 1.2309880190717277,
+      "learning_rate": 5.007669667551931e-08,
+      "loss": 1.7386,
+      "step": 1340
+    },
+    {
+      "epoch": 2.711693548387097,
+      "grad_norm": 1.2646430684773242,
+      "learning_rate": 5.007044163304792e-08,
+      "loss": 1.7271,
+      "step": 1345
+    },
+    {
+      "epoch": 2.721774193548387,
+      "grad_norm": 1.3573014483042332,
+      "learning_rate": 5.0064653838557546e-08,
+      "loss": 1.8166,
+      "step": 1350
+    },
+    {
+      "epoch": 2.7318548387096775,
+      "grad_norm": 1.260061171998465,
+      "learning_rate": 5.0059301641004814e-08,
+      "loss": 1.7825,
+      "step": 1355
+    },
+    {
+      "epoch": 2.741935483870968,
+      "grad_norm": 1.2930142933374957,
+      "learning_rate": 5.0054355309704326e-08,
+      "loss": 1.7362,
+      "step": 1360
+    },
+    {
+      "epoch": 2.752016129032258,
+      "grad_norm": 1.1917500201859248,
+      "learning_rate": 5.004978693159421e-08,
+      "loss": 1.7862,
+      "step": 1365
+    },
+    {
+      "epoch": 2.7620967741935485,
+      "grad_norm": 1.309363529031979,
+      "learning_rate": 5.004557031324913e-08,
+      "loss": 1.766,
+      "step": 1370
+    },
+    {
+      "epoch": 2.772177419354839,
+      "grad_norm": 1.260854474561999,
+      "learning_rate": 5.00416808874565e-08,
+      "loss": 1.7798,
+      "step": 1375
+    },
+    {
+      "epoch": 2.782258064516129,
+      "grad_norm": 1.2133948555271834,
+      "learning_rate": 5.003809562417745e-08,
+      "loss": 1.7702,
+      "step": 1380
+    },
+    {
+      "epoch": 2.7923387096774195,
+      "grad_norm": 1.20285403043514,
+      "learning_rate": 5.00347929457196e-08,
+      "loss": 1.7629,
+      "step": 1385
+    },
+    {
+      "epoch": 2.8024193548387095,
+      "grad_norm": 1.2055338414881442,
+      "learning_rate": 5.003175264595438e-08,
+      "loss": 1.6825,
+      "step": 1390
+    },
+    {
+      "epoch": 2.8125,
+      "grad_norm": 1.2692380612645446,
+      "learning_rate": 5.002895581341684e-08,
+      "loss": 1.7922,
+      "step": 1395
+    },
+    {
+      "epoch": 2.8225806451612905,
+      "grad_norm": 1.197125036474829,
+      "learning_rate": 5.002638475813119e-08,
+      "loss": 1.7346,
+      "step": 1400
+    },
+    {
+      "epoch": 2.8225806451612905,
+      "eval_loss": 1.9443275928497314,
+      "eval_runtime": 89.8463,
+      "eval_samples_per_second": 19.6,
+      "eval_steps_per_second": 0.312,
+      "step": 1400
+    },
+    {
+      "epoch": 2.8326612903225805,
+      "grad_norm": 1.3129336482449783,
+      "learning_rate": 5.0024022942010466e-08,
+      "loss": 1.7541,
+      "step": 1405
+    },
+    {
+      "epoch": 2.842741935483871,
+      "grad_norm": 1.1872922962263166,
+      "learning_rate": 5.002185491268355e-08,
+      "loss": 1.6476,
+      "step": 1410
+    },
+    {
+      "epoch": 2.852822580645161,
+      "grad_norm": 1.3099170695792106,
+      "learning_rate": 5.0019866240607806e-08,
+      "loss": 1.6647,
+      "step": 1415
+    },
+    {
+      "epoch": 2.8629032258064515,
+      "grad_norm": 1.3296060602873512,
+      "learning_rate": 5.001804345933021e-08,
+      "loss": 1.7079,
+      "step": 1420
+    },
+    {
+      "epoch": 2.872983870967742,
+      "grad_norm": 1.2481767829563293,
+      "learning_rate": 5.001637400876446e-08,
+      "loss": 1.6979,
+      "step": 1425
+    },
+    {
+      "epoch": 2.883064516129032,
+      "grad_norm": 1.3774040469968931,
+      "learning_rate": 5.001484618135618e-08,
+      "loss": 1.716,
+      "step": 1430
+    },
+    {
+      "epoch": 2.8931451612903225,
+      "grad_norm": 1.3300791138778125,
+      "learning_rate": 5.0013449071012536e-08,
+      "loss": 1.8281,
+      "step": 1435
+    },
+    {
+      "epoch": 2.903225806451613,
+      "grad_norm": 1.2648870887474146,
+      "learning_rate": 5.0012172524676996e-08,
+      "loss": 1.7511,
+      "step": 1440
+    },
+    {
+      "epoch": 2.913306451612903,
+      "grad_norm": 1.2324711482162307,
+      "learning_rate": 5.001100709643403e-08,
+      "loss": 1.7665,
+      "step": 1445
+    },
+    {
+      "epoch": 2.9233870967741935,
+      "grad_norm": 1.1884469998607576,
+      "learning_rate": 5.0009944004032616e-08,
+      "loss": 1.7515,
+      "step": 1450
+    },
+    {
+      "epoch": 2.933467741935484,
+      "grad_norm": 1.2242938574470985,
+      "learning_rate": 5.000897508772145e-08,
+      "loss": 1.779,
+      "step": 1455
+    },
+    {
+      "epoch": 2.943548387096774,
+      "grad_norm": 1.1767587628025173,
+      "learning_rate": 5.000809277129238e-08,
+      "loss": 1.7013,
+      "step": 1460
+    },
+    {
+      "epoch": 2.9536290322580645,
+      "grad_norm": 1.1876105747863372,
+      "learning_rate": 5.000729002523258e-08,
+      "loss": 1.7691,
+      "step": 1465
+    },
+    {
+      "epoch": 2.963709677419355,
+      "grad_norm": 1.2804911094505236,
+      "learning_rate": 5.0006560331889364e-08,
+      "loss": 1.7952,
+      "step": 1470
+    },
+    {
+      "epoch": 2.973790322580645,
+      "grad_norm": 1.2634767897263572,
+      "learning_rate": 5.0005897652555256e-08,
+      "loss": 1.8265,
+      "step": 1475
+    },
+    {
+      "epoch": 2.9838709677419355,
+      "grad_norm": 1.376696085159001,
+      "learning_rate": 5.000529639638423e-08,
+      "loss": 1.7974,
+      "step": 1480
+    },
+    {
+      "epoch": 2.993951612903226,
+      "grad_norm": 1.1674051041442062,
+      "learning_rate": 5.000475139105343e-08,
+      "loss": 1.6758,
+      "step": 1485
+    },
+    {
+      "epoch": 3.004032258064516,
+      "grad_norm": 1.2278436300934317,
+      "learning_rate": 5.000425785508798e-08,
+      "loss": 1.7076,
+      "step": 1490
+    },
+    {
+      "epoch": 3.0141129032258065,
+      "grad_norm": 1.208822859914408,
+      "learning_rate": 5.0003811371769436e-08,
+      "loss": 1.6804,
+      "step": 1495
+    },
+    {
+      "epoch": 3.024193548387097,
+      "grad_norm": 1.2607881481569827,
+      "learning_rate": 5.000340786455173e-08,
+      "loss": 1.7044,
+      "step": 1500
+    },
+    {
+      "epoch": 3.034274193548387,
+      "grad_norm": 1.2481468783314629,
+      "learning_rate": 5.00030435739113e-08,
+      "loss": 1.7133,
+      "step": 1505
+    },
+    {
+      "epoch": 3.0443548387096775,
+      "grad_norm": 1.2625933161048073,
+      "learning_rate": 5.0002715035560964e-08,
+      "loss": 1.7043,
+      "step": 1510
+    },
+    {
+      "epoch": 3.0544354838709675,
+      "grad_norm": 1.2211349887353309,
+      "learning_rate": 5.000241905995991e-08,
+      "loss": 1.7931,
+      "step": 1515
+    },
+    {
+      "epoch": 3.064516129032258,
+      "grad_norm": 1.2628888420799884,
+      "learning_rate": 5.000215271305494e-08,
+      "loss": 1.7648,
+      "step": 1520
+    },
+    {
+      "epoch": 3.0745967741935485,
+      "grad_norm": 1.2349226985107031,
+      "learning_rate": 5.000191329819063e-08,
+      "loss": 1.72,
+      "step": 1525
+    },
+    {
+      "epoch": 3.0846774193548385,
+      "grad_norm": 1.2993139941453224,
+      "learning_rate": 5.0001698339128735e-08,
+      "loss": 1.7493,
+      "step": 1530
+    },
+    {
+      "epoch": 3.094758064516129,
+      "grad_norm": 1.3021805987677582,
+      "learning_rate": 5.0001505564119494e-08,
+      "loss": 1.7801,
+      "step": 1535
+    },
+    {
+      "epoch": 3.1048387096774195,
+      "grad_norm": 1.2177089098452982,
+      "learning_rate": 5.000133289097e-08,
+      "loss": 1.7488,
+      "step": 1540
+    },
+    {
+      "epoch": 3.1149193548387095,
+      "grad_norm": 1.2475923991723818,
+      "learning_rate": 5.0001178413057135e-08,
+      "loss": 1.8035,
+      "step": 1545
+    },
+    {
+      "epoch": 3.125,
+      "grad_norm": 1.2750062841943113,
+      "learning_rate": 5.0001040386234535e-08,
+      "loss": 1.6927,
+      "step": 1550
+    },
+    {
+      "epoch": 3.1350806451612905,
+      "grad_norm": 1.2521384201728667,
+      "learning_rate": 5.000091721658573e-08,
+      "loss": 1.7867,
+      "step": 1555
+    },
+    {
+      "epoch": 3.1451612903225805,
+      "grad_norm": 1.2788389170185523,
+      "learning_rate": 5.0000807448977166e-08,
+      "loss": 1.7463,
+      "step": 1560
+    },
+    {
+      "epoch": 3.155241935483871,
+      "grad_norm": 1.1932742831754912,
+      "learning_rate": 5.000070975636714e-08,
+      "loss": 1.6646,
+      "step": 1565
+    },
+    {
+      "epoch": 3.1653225806451615,
+      "grad_norm": 1.2026279996869336,
+      "learning_rate": 5.0000622929828655e-08,
+      "loss": 1.7426,
+      "step": 1570
+    },
+    {
+      "epoch": 3.1754032258064515,
+      "grad_norm": 1.2658970944840857,
+      "learning_rate": 5.000054586924596e-08,
+      "loss": 1.6139,
+      "step": 1575
+    },
+    {
+      "epoch": 3.185483870967742,
+      "grad_norm": 1.2582895003633043,
+      "learning_rate": 5.0000477574646376e-08,
+      "loss": 1.6655,
+      "step": 1580
+    },
+    {
+      "epoch": 3.195564516129032,
+      "grad_norm": 1.286929029486268,
+      "learning_rate": 5.000041713813098e-08,
+      "loss": 1.8082,
+      "step": 1585
+    },
+    {
+      "epoch": 3.2056451612903225,
+      "grad_norm": 1.5242979315943919,
+      "learning_rate": 5.0000363736369096e-08,
+      "loss": 1.7129,
+      "step": 1590
+    },
+    {
+      "epoch": 3.215725806451613,
+      "grad_norm": 1.1974095695410052,
+      "learning_rate": 5.000031662362353e-08,
+      "loss": 1.7858,
+      "step": 1595
+    },
+    {
+      "epoch": 3.225806451612903,
+      "grad_norm": 1.2334951186835896,
+      "learning_rate": 5.000027512527476e-08,
+      "loss": 1.7028,
+      "step": 1600
+    },
+    {
+      "epoch": 3.225806451612903,
+      "eval_loss": 1.9448145627975464,
+      "eval_runtime": 89.9094,
+      "eval_samples_per_second": 19.586,
+      "eval_steps_per_second": 0.311,
+      "step": 1600
+    },
+    {
+      "epoch": 3.2358870967741935,
+      "grad_norm": 1.2639640178197116,
+      "learning_rate": 5.0000238631814066e-08,
+      "loss": 1.8326,
+      "step": 1605
+    },
+    {
+      "epoch": 3.245967741935484,
+      "grad_norm": 1.2369747127772903,
+      "learning_rate": 5.000020659327692e-08,
+      "loss": 1.6435,
+      "step": 1610
+    },
+    {
+      "epoch": 3.256048387096774,
+      "grad_norm": 1.2345433192530466,
+      "learning_rate": 5.000017851408939e-08,
+      "loss": 1.7267,
+      "step": 1615
+    },
+    {
+      "epoch": 3.2661290322580645,
+      "grad_norm": 1.2504468827628226,
+      "learning_rate": 5.0000153948301755e-08,
+      "loss": 1.749,
+      "step": 1620
+    },
+    {
+      "epoch": 3.276209677419355,
+      "grad_norm": 1.2484879937766102,
+      "learning_rate": 5.000013249518468e-08,
+      "loss": 1.7147,
+      "step": 1625
+    },
+    {
+      "epoch": 3.286290322580645,
+      "grad_norm": 1.2765939466692027,
+      "learning_rate": 5.000011379516478e-08,
+      "loss": 1.708,
+      "step": 1630
+    },
+    {
+      "epoch": 3.2963709677419355,
+      "grad_norm": 1.2548830946050018,
+      "learning_rate": 5.000009752607744e-08,
+      "loss": 1.6576,
+      "step": 1635
+    },
+    {
+      "epoch": 3.306451612903226,
+      "grad_norm": 1.3380090814755392,
+      "learning_rate": 5.00000833997159e-08,
+      "loss": 1.7899,
+      "step": 1640
+    },
+    {
+      "epoch": 3.316532258064516,
+      "grad_norm": 1.2589567647915303,
+      "learning_rate": 5.000007115865701e-08,
+      "loss": 1.7283,
+      "step": 1645
+    },
+    {
+      "epoch": 3.3266129032258065,
+      "grad_norm": 1.330891571705861,
+      "learning_rate": 5.00000605733447e-08,
+      "loss": 1.7638,
+      "step": 1650
+    },
+    {
+      "epoch": 3.336693548387097,
+      "grad_norm": 1.2968430137554552,
+      "learning_rate": 5.0000051439413665e-08,
+      "loss": 1.7894,
+      "step": 1655
+    },
+    {
+      "epoch": 3.346774193548387,
+      "grad_norm": 1.26655914538691,
+      "learning_rate": 5.000004357523643e-08,
+      "loss": 1.6794,
+      "step": 1660
+    },
+    {
+      "epoch": 3.3568548387096775,
+      "grad_norm": 1.3255247557489993,
+      "learning_rate": 5.000003681967807e-08,
+      "loss": 1.7458,
+      "step": 1665
+    },
+    {
+      "epoch": 3.366935483870968,
+      "grad_norm": 1.4616051241449233,
+      "learning_rate": 5.000003103004378e-08,
+      "loss": 1.7682,
+      "step": 1670
+    },
+    {
+      "epoch": 3.377016129032258,
+      "grad_norm": 1.2909185531222274,
+      "learning_rate": 5.000002608020516e-08,
+      "loss": 1.7676,
+      "step": 1675
+    },
+    {
+      "epoch": 3.3870967741935485,
+      "grad_norm": 1.5116117360689252,
+      "learning_rate": 5.0000021858892204e-08,
+      "loss": 1.8041,
+      "step": 1680
+    },
+    {
+      "epoch": 3.3971774193548385,
+      "grad_norm": 1.2019942003042767,
+      "learning_rate": 5.000001826813845e-08,
+      "loss": 1.6975,
+      "step": 1685
+    },
+    {
+      "epoch": 3.407258064516129,
+      "grad_norm": 1.2423004849326869,
+      "learning_rate": 5.000001522186781e-08,
+      "loss": 1.7856,
+      "step": 1690
+    },
+    {
+      "epoch": 3.4173387096774195,
+      "grad_norm": 1.1833866067645467,
+      "learning_rate": 5.0000012644612046e-08,
+      "loss": 1.6801,
+      "step": 1695
+    },
+    {
+      "epoch": 3.4274193548387095,
+      "grad_norm": 1.2527324283211838,
+      "learning_rate": 5.000001047034869e-08,
+      "loss": 1.7829,
+      "step": 1700
+    },
+    {
+      "epoch": 3.4375,
+      "grad_norm": 1.2021893152164589,
+      "learning_rate": 5.000000864144987e-08,
+      "loss": 1.7365,
+      "step": 1705
+    },
+    {
+      "epoch": 3.4475806451612905,
+      "grad_norm": 1.2533733399310467,
+      "learning_rate": 5.0000007107733085e-08,
+      "loss": 1.7684,
+      "step": 1710
+    },
+    {
+      "epoch": 3.4576612903225805,
+      "grad_norm": 1.2808992378124462,
+      "learning_rate": 5.000000582560542e-08,
+      "loss": 1.7195,
+      "step": 1715
+    },
+    {
+      "epoch": 3.467741935483871,
+      "grad_norm": 1.2244209413905693,
+      "learning_rate": 5.00000047572936e-08,
+      "loss": 1.7485,
+      "step": 1720
+    },
+    {
+      "epoch": 3.4778225806451615,
+      "grad_norm": 1.2540825243078901,
+      "learning_rate": 5.000000387015242e-08,
+      "loss": 1.7303,
+      "step": 1725
+    },
+    {
+      "epoch": 3.4879032258064515,
+      "grad_norm": 1.2847488818819852,
+      "learning_rate": 5.0000003136044955e-08,
+      "loss": 1.8182,
+      "step": 1730
+    },
+    {
+      "epoch": 3.497983870967742,
+      "grad_norm": 1.247272423872035,
+      "learning_rate": 5.000000253078795e-08,
+      "loss": 1.7646,
+      "step": 1735
+    },
+    {
+      "epoch": 3.508064516129032,
+      "grad_norm": 1.1616622600983573,
+      "learning_rate": 5.000000203365694e-08,
+      "loss": 1.61,
+      "step": 1740
+    },
+    {
+      "epoch": 3.5181451612903225,
+      "grad_norm": 1.2602897978426342,
+      "learning_rate": 5.000000162694535e-08,
+      "loss": 1.7119,
+      "step": 1745
+    },
+    {
+      "epoch": 3.528225806451613,
+      "grad_norm": 1.2351778503899846,
+      "learning_rate": 5.000000129557269e-08,
+      "loss": 1.7027,
+      "step": 1750
+    },
+    {
+      "epoch": 3.538306451612903,
+      "grad_norm": 1.2344538732881385,
+      "learning_rate": 5.000000102673722e-08,
+      "loss": 1.7228,
+      "step": 1755
+    },
+    {
+      "epoch": 3.5483870967741935,
+      "grad_norm": 1.2335083557737399,
+      "learning_rate": 5.0000000809608727e-08,
+      "loss": 1.7107,
+      "step": 1760
+    },
+    {
+      "epoch": 3.558467741935484,
+      "grad_norm": 1.2491739462049536,
+      "learning_rate": 5.000000063505754e-08,
+      "loss": 1.6427,
+      "step": 1765
+    },
+    {
+      "epoch": 3.568548387096774,
+      "grad_norm": 1.2486316865372293,
+      "learning_rate": 5.000000049541612e-08,
+      "loss": 1.7365,
+      "step": 1770
+    },
+    {
+      "epoch": 3.5786290322580645,
+      "grad_norm": 1.3051302158136038,
+      "learning_rate": 5.000000038426991e-08,
+      "loss": 1.7311,
+      "step": 1775
+    },
+    {
+      "epoch": 3.588709677419355,
+      "grad_norm": 1.23592969040822,
+      "learning_rate": 5.000000029627433e-08,
+      "loss": 1.7672,
+      "step": 1780
+    },
+    {
+      "epoch": 3.598790322580645,
+      "grad_norm": 1.1961157739658335,
+      "learning_rate": 5.000000022699522e-08,
+      "loss": 1.6726,
+      "step": 1785
+    },
+    {
+      "epoch": 3.6088709677419355,
+      "grad_norm": 1.2487972857881837,
+      "learning_rate": 5.000000017277015e-08,
+      "loss": 1.823,
+      "step": 1790
+    },
+    {
+      "epoch": 3.618951612903226,
+      "grad_norm": 1.2187655485543294,
+      "learning_rate": 5.0000000130588186e-08,
+      "loss": 1.7657,
+      "step": 1795
+    },
+    {
+      "epoch": 3.629032258064516,
+      "grad_norm": 1.196141679163838,
+      "learning_rate": 5.000000009798615e-08,
+      "loss": 1.7383,
+      "step": 1800
+    },
+    {
+      "epoch": 3.629032258064516,
+      "eval_loss": 1.9450066089630127,
+      "eval_runtime": 89.836,
+      "eval_samples_per_second": 19.602,
+      "eval_steps_per_second": 0.312,
+      "step": 1800
+    },
+    {
+      "epoch": 3.6391129032258065,
+      "grad_norm": 1.2843403341661654,
+      "learning_rate": 5.0000000072959346e-08,
+      "loss": 1.6978,
+      "step": 1805
+    },
+    {
+      "epoch": 3.649193548387097,
+      "grad_norm": 1.2729441010270168,
+      "learning_rate": 5.000000005388504e-08,
+      "loss": 1.7236,
+      "step": 1810
+    },
+    {
+      "epoch": 3.659274193548387,
+      "grad_norm": 1.278805125989498,
+      "learning_rate": 5.000000003945723e-08,
+      "loss": 1.7535,
+      "step": 1815
+    },
+    {
+      "epoch": 3.6693548387096775,
+      "grad_norm": 1.3166466197064246,
+      "learning_rate": 5.0000000028631134e-08,
+      "loss": 1.7694,
+      "step": 1820
+    },
+    {
+      "epoch": 3.679435483870968,
+      "grad_norm": 1.2114793447075247,
+      "learning_rate": 5.0000000020576294e-08,
+      "loss": 1.7647,
+      "step": 1825
+    },
+    {
+      "epoch": 3.689516129032258,
+      "grad_norm": 1.2395966902447142,
+      "learning_rate": 5.000000001463704e-08,
+      "loss": 1.7112,
+      "step": 1830
+    },
+    {
+      "epoch": 3.6995967741935485,
+      "grad_norm": 1.2716522150792229,
+      "learning_rate": 5.0000000010299415e-08,
+      "loss": 1.7674,
+      "step": 1835
+    },
+    {
+      "epoch": 3.709677419354839,
+      "grad_norm": 1.2205432657380169,
+      "learning_rate": 5.000000000716364e-08,
+      "loss": 1.6942,
+      "step": 1840
+    },
+    {
+      "epoch": 3.719758064516129,
+      "grad_norm": 1.3306099794060742,
+      "learning_rate": 5.000000000492121e-08,
+      "loss": 1.6627,
+      "step": 1845
+    },
+    {
+      "epoch": 3.7298387096774195,
+      "grad_norm": 1.2101658596755476,
+      "learning_rate": 5.000000000333616e-08,
+      "loss": 1.719,
+      "step": 1850
+    },
+    {
+      "epoch": 3.7399193548387095,
+      "grad_norm": 1.2500275205533455,
+      "learning_rate": 5.000000000222965e-08,
+      "loss": 1.7493,
+      "step": 1855
+    },
+    {
+      "epoch": 3.75,
+      "grad_norm": 1.2312257846231576,
+      "learning_rate": 5.0000000001467476e-08,
+      "loss": 1.7789,
+      "step": 1860
+    },
+    {
+      "epoch": 3.7600806451612905,
+      "grad_norm": 1.2152299183587771,
+      "learning_rate": 5.000000000095e-08,
+      "loss": 1.7211,
+      "step": 1865
+    },
+    {
+      "epoch": 3.7701612903225805,
+      "grad_norm": 1.1996585863358138,
+      "learning_rate": 5.0000000000604084e-08,
+      "loss": 1.7994,
+      "step": 1870
+    },
+    {
+      "epoch": 3.780241935483871,
+      "grad_norm": 1.3230860176481571,
+      "learning_rate": 5.0000000000376735e-08,
+      "loss": 1.7701,
+      "step": 1875
+    },
+    {
+      "epoch": 3.790322580645161,
+      "grad_norm": 1.2748379623445445,
+      "learning_rate": 5.000000000023003e-08,
+      "loss": 1.7793,
+      "step": 1880
+    },
+    {
+      "epoch": 3.8004032258064515,
+      "grad_norm": 1.2509667612393764,
+      "learning_rate": 5.000000000013723e-08,
+      "loss": 1.7706,
+      "step": 1885
+    },
+    {
+      "epoch": 3.810483870967742,
+      "grad_norm": 1.2694885460408405,
+      "learning_rate": 5.000000000007981e-08,
+      "loss": 1.7397,
+      "step": 1890
+    },
+    {
+      "epoch": 3.820564516129032,
+      "grad_norm": 1.3091515145607215,
+      "learning_rate": 5.000000000004513e-08,
+      "loss": 1.7211,
+      "step": 1895
+    },
+    {
+      "epoch": 3.8306451612903225,
+      "grad_norm": 1.247922300219581,
+      "learning_rate": 5.0000000000024734e-08,
+      "loss": 1.774,
+      "step": 1900
+    },
+    {
+      "epoch": 3.840725806451613,
+      "grad_norm": 1.2523345160639314,
+      "learning_rate": 5.0000000000013094e-08,
+      "loss": 1.6928,
+      "step": 1905
+    },
+    {
+      "epoch": 3.850806451612903,
+      "grad_norm": 1.1986290945949785,
+      "learning_rate": 5.000000000000667e-08,
+      "loss": 1.7161,
+      "step": 1910
+    },
+    {
+      "epoch": 3.8608870967741935,
+      "grad_norm": 1.2245878014293898,
+      "learning_rate": 5.000000000000325e-08,
+      "loss": 1.686,
+      "step": 1915
+    },
+    {
+      "epoch": 3.870967741935484,
+      "grad_norm": 1.2249127134236595,
+      "learning_rate": 5.00000000000015e-08,
+      "loss": 1.7206,
+      "step": 1920
+    },
+    {
+      "epoch": 3.881048387096774,
+      "grad_norm": 1.3558345618343015,
+      "learning_rate": 5.000000000000065e-08,
+      "loss": 1.68,
+      "step": 1925
+    },
+    {
+      "epoch": 3.8911290322580645,
+      "grad_norm": 1.3236906608919645,
+      "learning_rate": 5.000000000000027e-08,
+      "loss": 1.6661,
+      "step": 1930
+    },
+    {
+      "epoch": 3.901209677419355,
+      "grad_norm": 1.183426329240457,
+      "learning_rate": 5.00000000000001e-08,
+      "loss": 1.727,
+      "step": 1935
+    },
+    {
+      "epoch": 3.911290322580645,
+      "grad_norm": 1.2348171369390457,
+      "learning_rate": 5.000000000000003e-08,
+      "loss": 1.6334,
+      "step": 1940
+    },
+    {
+      "epoch": 3.9213709677419355,
+      "grad_norm": 1.2911302775087936,
+      "learning_rate": 5.000000000000001e-08,
+      "loss": 1.8057,
+      "step": 1945
+    },
+    {
+      "epoch": 3.931451612903226,
+      "grad_norm": 1.1863076460974697,
+      "learning_rate": 5e-08,
+      "loss": 1.6921,
+      "step": 1950
+    },
+    {
+      "epoch": 3.941532258064516,
+      "grad_norm": 1.2618121531154096,
+      "learning_rate": 5e-08,
+      "loss": 1.6586,
+      "step": 1955
+    },
+    {
+      "epoch": 3.9516129032258065,
+      "grad_norm": 1.4241738579497785,
+      "learning_rate": 5e-08,
+      "loss": 1.7724,
+      "step": 1960
+    },
+    {
+      "epoch": 3.961693548387097,
+      "grad_norm": 1.3305658271138237,
+      "learning_rate": 5e-08,
+      "loss": 1.759,
+      "step": 1965
+    },
+    {
+      "epoch": 3.971774193548387,
+      "grad_norm": 1.364160190258584,
+      "learning_rate": 5e-08,
+      "loss": 1.7014,
+      "step": 1970
+    },
+    {
+      "epoch": 3.9818548387096775,
+      "grad_norm": 1.206231989148295,
+      "learning_rate": 5e-08,
+      "loss": 1.6474,
+      "step": 1975
+    },
+    {
+      "epoch": 3.991935483870968,
+      "grad_norm": 1.3115654648816009,
+      "learning_rate": 5e-08,
+      "loss": 1.7559,
+      "step": 1980
+    },
+    {
+      "epoch": 4.0,
+      "step": 1984,
+      "total_flos": 414362338590720.0,
+      "train_loss": 1.7886004945443523,
+      "train_runtime": 13163.6015,
+      "train_samples_per_second": 4.815,
+      "train_steps_per_second": 0.151
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1984,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 200,
+  "total_flos": 414362338590720.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83c783c735b64736a3045bb52877c6e032db97cfdb1cacff2f7cb0139fcaf52c
+size 6840