Upload folder using huggingface_hub

README.md

@@ -0,0 +1,44 @@
+---
+tags:
+- autotrain
+- text-generation-inference
+- text-generation
+library_name: transformers
+widget:
+  - messages:
+      - role: user
+        content: What is your favorite condiment?
+license: other
+---
+
+# Model Trained Using AutoTrain
+
+This model was trained using AutoTrain. For more information, please visit [AutoTrain](https://hf.co/docs/autotrain).
+
+# Usage
+
+```python
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_path = "PATH_TO_THIS_REPO"
+
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+model = AutoModelForCausalLM.from_pretrained(
+    model_path,
+    device_map="auto",
+    torch_dtype='auto'
+).eval()
+
+# Prompt content: "hi"
+messages = [
+    {"role": "user", "content": "hi"}
+]
+
+input_ids = tokenizer.apply_chat_template(conversation=messages, tokenize=True, add_generation_prompt=True, return_tensors='pt')
+output_ids = model.generate(input_ids.to('cuda'))
+response = tokenizer.decode(output_ids[0][input_ids.shape[1]:], skip_special_tokens=True)
+
+# Model response: "Hello! How can I assist you today?"
+print(response)
+```

checkpoint-11153/config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "meta-llama/Meta-Llama-3-8B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.0",
+  "use_cache": false,
+  "vocab_size": 128264
+}

checkpoint-11153/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "transformers_version": "4.40.0"
+}

checkpoint-11153/latest

@@ -0,0 +1 @@
+global_step11153

checkpoint-11153/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4dc60ba1d2073bdd95a0f469265587e9711e0e35b5569da784f287cd6ab0356
+size 4976764208

checkpoint-11153/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cb22b1acddca151372ab19dda1bc7dfed4848c70e15bf93e20fc7c02b4fdb98
+size 4999802720

checkpoint-11153/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5136aa7410834a6af82d77dbbb7b3e854b09b750e55406a7c6b3ced0e24b4afb
+size 4915916176

checkpoint-11153/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8708006d0d25d48da829f87f16dd42ba10f7286b63ab292b7546c45be3ff2059
+size 1168204344

checkpoint-11153/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16060653568
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-11153/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:575119a228f98110923ffa2dedcb50e3317251b26054355d015e0b2240d566f2
+size 15984

checkpoint-11153/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0728b56dab7abb5ef8a0d4bae3519c5767c97467bdd886d26bf19cc8599d0312
+size 15984

checkpoint-11153/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4e481d4ef1546694da7337f6bb6c658b866dcb79b85deeb477da0d27ebe851e
+size 15984

checkpoint-11153/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:353c60be37ea56fc992fca446598ceca5d1fd002aa3bd6dbb9ad740e6f47ebb3
+size 15984

checkpoint-11153/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9107fe964ba7205e354084b85210e5a5ea1c98cfd4d38adb9cd3926945dcae4
+size 15984

checkpoint-11153/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69d1bb1abee38b92e53f3f23549b642ce0f1edcdccf7b6129847ac61636e96d5
+size 15984

checkpoint-11153/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afd5516048e20f36959601574e29e40106085a7d3cdc7bf425ce5e84633490e6
+size 15984

checkpoint-11153/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e2c46927fc06939b4c976a01e4b95dec1f8b98ceaea86d31a5d756fc30ff006
+size 15984

checkpoint-11153/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d960e8208aff0ce325af0e98a7691007c04a2bd8d6ea631be6bf97d32f9ca866
+size 1064

checkpoint-11153/special_tokens_map.json

@@ -0,0 +1,13 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>user",
+    "<|im_start|>assistant",
+    "<|im_start|>system",
+    "<|im_end|>",
+    "<s>",
+    "<pad>"
+  ],
+  "bos_token": "<s>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<pad>"
+}

checkpoint-11153/tokenizer_config.json

@@ -0,0 +1,2119 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128256": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128257": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128258": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128259": {
+      "content": "<|im_start|>user",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128260": {
+      "content": "<|im_start|>assistant",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128261": {
+      "content": "<|im_start|>system",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>user",
+    "<|im_start|>assistant",
+    "<|im_start|>system",
+    "<|im_end|>",
+    "<s>",
+    "<pad>"
+  ],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% if loop.last and add_generation_prompt %}{{'<|im_start|>assistant\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|im_end|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 8192,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-11153/trainer_state.json

@@ -0,0 +1,3143 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 11153,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0022415493589168834,
+      "grad_norm": 15.685082226240027,
+      "learning_rate": 4.480286738351255e-07,
+      "loss": 1.726,
+      "step": 25
+    },
+    {
+      "epoch": 0.004483098717833767,
+      "grad_norm": 4.767375275306418,
+      "learning_rate": 8.96057347670251e-07,
+      "loss": 1.4821,
+      "step": 50
+    },
+    {
+      "epoch": 0.00672464807675065,
+      "grad_norm": 2.7898427654813496,
+      "learning_rate": 1.3440860215053765e-06,
+      "loss": 1.3178,
+      "step": 75
+    },
+    {
+      "epoch": 0.008966197435667534,
+      "grad_norm": 2.450767559139303,
+      "learning_rate": 1.792114695340502e-06,
+      "loss": 1.2526,
+      "step": 100
+    },
+    {
+      "epoch": 0.011207746794584417,
+      "grad_norm": 2.1230121564392475,
+      "learning_rate": 2.2401433691756277e-06,
+      "loss": 1.2204,
+      "step": 125
+    },
+    {
+      "epoch": 0.0134492961535013,
+      "grad_norm": 2.5183780437270205,
+      "learning_rate": 2.688172043010753e-06,
+      "loss": 1.1894,
+      "step": 150
+    },
+    {
+      "epoch": 0.015690845512418183,
+      "grad_norm": 2.1917851710152974,
+      "learning_rate": 3.1362007168458786e-06,
+      "loss": 1.1946,
+      "step": 175
+    },
+    {
+      "epoch": 0.017932394871335067,
+      "grad_norm": 2.2162729922708118,
+      "learning_rate": 3.584229390681004e-06,
+      "loss": 1.1389,
+      "step": 200
+    },
+    {
+      "epoch": 0.02017394423025195,
+      "grad_norm": 2.3941641310533925,
+      "learning_rate": 4.032258064516129e-06,
+      "loss": 1.1478,
+      "step": 225
+    },
+    {
+      "epoch": 0.022415493589168833,
+      "grad_norm": 3.072121988145324,
+      "learning_rate": 4.480286738351255e-06,
+      "loss": 1.1468,
+      "step": 250
+    },
+    {
+      "epoch": 0.024657042948085718,
+      "grad_norm": 2.219661451743802,
+      "learning_rate": 4.928315412186381e-06,
+      "loss": 1.1425,
+      "step": 275
+    },
+    {
+      "epoch": 0.0268985923070026,
+      "grad_norm": 2.4345631665676533,
+      "learning_rate": 5.376344086021506e-06,
+      "loss": 1.1095,
+      "step": 300
+    },
+    {
+      "epoch": 0.029140141665919484,
+      "grad_norm": 2.047942477418226,
+      "learning_rate": 5.824372759856631e-06,
+      "loss": 1.1204,
+      "step": 325
+    },
+    {
+      "epoch": 0.031381691024836365,
+      "grad_norm": 2.175499992275822,
+      "learning_rate": 6.272401433691757e-06,
+      "loss": 1.1202,
+      "step": 350
+    },
+    {
+      "epoch": 0.03362324038375325,
+      "grad_norm": 2.25503906485662,
+      "learning_rate": 6.720430107526882e-06,
+      "loss": 1.1022,
+      "step": 375
+    },
+    {
+      "epoch": 0.035864789742670135,
+      "grad_norm": 2.108460216356371,
+      "learning_rate": 7.168458781362008e-06,
+      "loss": 1.0877,
+      "step": 400
+    },
+    {
+      "epoch": 0.03810633910158702,
+      "grad_norm": 2.240064238054789,
+      "learning_rate": 7.616487455197133e-06,
+      "loss": 1.1007,
+      "step": 425
+    },
+    {
+      "epoch": 0.0403478884605039,
+      "grad_norm": 2.4263093751968734,
+      "learning_rate": 8.064516129032258e-06,
+      "loss": 1.0936,
+      "step": 450
+    },
+    {
+      "epoch": 0.04258943781942078,
+      "grad_norm": 2.387017798068924,
+      "learning_rate": 8.512544802867385e-06,
+      "loss": 1.0761,
+      "step": 475
+    },
+    {
+      "epoch": 0.04483098717833767,
+      "grad_norm": 2.1179318054101564,
+      "learning_rate": 8.96057347670251e-06,
+      "loss": 1.0771,
+      "step": 500
+    },
+    {
+      "epoch": 0.04707253653725455,
+      "grad_norm": 1.9922816447378509,
+      "learning_rate": 9.408602150537635e-06,
+      "loss": 1.0864,
+      "step": 525
+    },
+    {
+      "epoch": 0.049314085896171436,
+      "grad_norm": 2.030735040176059,
+      "learning_rate": 9.856630824372761e-06,
+      "loss": 1.0932,
+      "step": 550
+    },
+    {
+      "epoch": 0.051555635255088314,
+      "grad_norm": 1.9365105060421755,
+      "learning_rate": 1.0304659498207886e-05,
+      "loss": 1.0992,
+      "step": 575
+    },
+    {
+      "epoch": 0.0537971846140052,
+      "grad_norm": 2.1272853307800044,
+      "learning_rate": 1.0752688172043012e-05,
+      "loss": 1.0712,
+      "step": 600
+    },
+    {
+      "epoch": 0.056038733972922083,
+      "grad_norm": 2.1859906837121907,
+      "learning_rate": 1.1200716845878136e-05,
+      "loss": 1.0748,
+      "step": 625
+    },
+    {
+      "epoch": 0.05828028333183897,
+      "grad_norm": 2.079572861691428,
+      "learning_rate": 1.1648745519713262e-05,
+      "loss": 1.0771,
+      "step": 650
+    },
+    {
+      "epoch": 0.06052183269075585,
+      "grad_norm": 1.9857796074700236,
+      "learning_rate": 1.2096774193548388e-05,
+      "loss": 1.0541,
+      "step": 675
+    },
+    {
+      "epoch": 0.06276338204967273,
+      "grad_norm": 2.0507709686842315,
+      "learning_rate": 1.2544802867383515e-05,
+      "loss": 1.0751,
+      "step": 700
+    },
+    {
+      "epoch": 0.06500493140858962,
+      "grad_norm": 1.952012882655032,
+      "learning_rate": 1.299283154121864e-05,
+      "loss": 1.0628,
+      "step": 725
+    },
+    {
+      "epoch": 0.0672464807675065,
+      "grad_norm": 2.1878324461110856,
+      "learning_rate": 1.3440860215053763e-05,
+      "loss": 1.0584,
+      "step": 750
+    },
+    {
+      "epoch": 0.06948803012642338,
+      "grad_norm": 2.054184787294387,
+      "learning_rate": 1.388888888888889e-05,
+      "loss": 1.0871,
+      "step": 775
+    },
+    {
+      "epoch": 0.07172957948534027,
+      "grad_norm": 2.125679946407211,
+      "learning_rate": 1.4336917562724016e-05,
+      "loss": 1.0784,
+      "step": 800
+    },
+    {
+      "epoch": 0.07397112884425715,
+      "grad_norm": 2.1121699070033872,
+      "learning_rate": 1.4784946236559142e-05,
+      "loss": 1.0598,
+      "step": 825
+    },
+    {
+      "epoch": 0.07621267820317404,
+      "grad_norm": 2.0730206555235906,
+      "learning_rate": 1.5232974910394266e-05,
+      "loss": 1.0786,
+      "step": 850
+    },
+    {
+      "epoch": 0.07845422756209092,
+      "grad_norm": 2.027127840079525,
+      "learning_rate": 1.5681003584229394e-05,
+      "loss": 1.0634,
+      "step": 875
+    },
+    {
+      "epoch": 0.0806957769210078,
+      "grad_norm": 2.023384971151795,
+      "learning_rate": 1.6129032258064517e-05,
+      "loss": 1.0764,
+      "step": 900
+    },
+    {
+      "epoch": 0.08293732627992469,
+      "grad_norm": 2.0055801278407075,
+      "learning_rate": 1.6577060931899643e-05,
+      "loss": 1.0673,
+      "step": 925
+    },
+    {
+      "epoch": 0.08517887563884156,
+      "grad_norm": 1.9506409683212098,
+      "learning_rate": 1.702508960573477e-05,
+      "loss": 1.0742,
+      "step": 950
+    },
+    {
+      "epoch": 0.08742042499775846,
+      "grad_norm": 2.1771441973840733,
+      "learning_rate": 1.7473118279569895e-05,
+      "loss": 1.0761,
+      "step": 975
+    },
+    {
+      "epoch": 0.08966197435667533,
+      "grad_norm": 2.011510686003709,
+      "learning_rate": 1.792114695340502e-05,
+      "loss": 1.0706,
+      "step": 1000
+    },
+    {
+      "epoch": 0.09190352371559221,
+      "grad_norm": 1.8219073311702734,
+      "learning_rate": 1.8369175627240144e-05,
+      "loss": 1.0867,
+      "step": 1025
+    },
+    {
+      "epoch": 0.0941450730745091,
+      "grad_norm": 2.084440231532221,
+      "learning_rate": 1.881720430107527e-05,
+      "loss": 1.1018,
+      "step": 1050
+    },
+    {
+      "epoch": 0.09638662243342598,
+      "grad_norm": 1.998527691194371,
+      "learning_rate": 1.9265232974910396e-05,
+      "loss": 1.0858,
+      "step": 1075
+    },
+    {
+      "epoch": 0.09862817179234287,
+      "grad_norm": 2.0128555781491784,
+      "learning_rate": 1.9713261648745522e-05,
+      "loss": 1.0525,
+      "step": 1100
+    },
+    {
+      "epoch": 0.10086972115125975,
+      "grad_norm": 2.080552489821866,
+      "learning_rate": 1.9982066354488394e-05,
+      "loss": 1.0685,
+      "step": 1125
+    },
+    {
+      "epoch": 0.10311127051017663,
+      "grad_norm": 2.093067690460913,
+      "learning_rate": 1.9932250672511707e-05,
+      "loss": 1.0376,
+      "step": 1150
+    },
+    {
+      "epoch": 0.10535281986909352,
+      "grad_norm": 1.9149860215931092,
+      "learning_rate": 1.9882434990535024e-05,
+      "loss": 1.0782,
+      "step": 1175
+    },
+    {
+      "epoch": 0.1075943692280104,
+      "grad_norm": 1.88017980052791,
+      "learning_rate": 1.9832619308558337e-05,
+      "loss": 1.0766,
+      "step": 1200
+    },
+    {
+      "epoch": 0.10983591858692729,
+      "grad_norm": 1.91492994234324,
+      "learning_rate": 1.978280362658165e-05,
+      "loss": 1.0922,
+      "step": 1225
+    },
+    {
+      "epoch": 0.11207746794584417,
+      "grad_norm": 2.0110976065228345,
+      "learning_rate": 1.9732987944604963e-05,
+      "loss": 1.0442,
+      "step": 1250
+    },
+    {
+      "epoch": 0.11431901730476104,
+      "grad_norm": 1.963223650876964,
+      "learning_rate": 1.9683172262628276e-05,
+      "loss": 1.0587,
+      "step": 1275
+    },
+    {
+      "epoch": 0.11656056666367794,
+      "grad_norm": 2.134419169579966,
+      "learning_rate": 1.963335658065159e-05,
+      "loss": 1.0926,
+      "step": 1300
+    },
+    {
+      "epoch": 0.11880211602259481,
+      "grad_norm": 1.9854410848170396,
+      "learning_rate": 1.9583540898674905e-05,
+      "loss": 1.0642,
+      "step": 1325
+    },
+    {
+      "epoch": 0.1210436653815117,
+      "grad_norm": 2.1190303504427717,
+      "learning_rate": 1.953372521669822e-05,
+      "loss": 1.0963,
+      "step": 1350
+    },
+    {
+      "epoch": 0.12328521474042858,
+      "grad_norm": 2.0917025707015915,
+      "learning_rate": 1.948390953472153e-05,
+      "loss": 1.0652,
+      "step": 1375
+    },
+    {
+      "epoch": 0.12552676409934546,
+      "grad_norm": 2.0833056932137466,
+      "learning_rate": 1.9434093852744848e-05,
+      "loss": 1.0774,
+      "step": 1400
+    },
+    {
+      "epoch": 0.12776831345826234,
+      "grad_norm": 1.923749930785264,
+      "learning_rate": 1.938427817076816e-05,
+      "loss": 1.066,
+      "step": 1425
+    },
+    {
+      "epoch": 0.13000986281717924,
+      "grad_norm": 1.917103256862209,
+      "learning_rate": 1.9334462488791474e-05,
+      "loss": 1.052,
+      "step": 1450
+    },
+    {
+      "epoch": 0.13225141217609612,
+      "grad_norm": 2.2817156042334896,
+      "learning_rate": 1.9284646806814787e-05,
+      "loss": 1.053,
+      "step": 1475
+    },
+    {
+      "epoch": 0.134492961535013,
+      "grad_norm": 1.8228317130823313,
+      "learning_rate": 1.92348311248381e-05,
+      "loss": 1.0346,
+      "step": 1500
+    },
+    {
+      "epoch": 0.13673451089392988,
+      "grad_norm": 1.8523579647705695,
+      "learning_rate": 1.9185015442861413e-05,
+      "loss": 1.0457,
+      "step": 1525
+    },
+    {
+      "epoch": 0.13897606025284676,
+      "grad_norm": 1.8768486959088566,
+      "learning_rate": 1.913519976088473e-05,
+      "loss": 1.0348,
+      "step": 1550
+    },
+    {
+      "epoch": 0.14121760961176366,
+      "grad_norm": 1.9857424877457883,
+      "learning_rate": 1.9085384078908043e-05,
+      "loss": 1.0439,
+      "step": 1575
+    },
+    {
+      "epoch": 0.14345915897068054,
+      "grad_norm": 2.531831058245927,
+      "learning_rate": 1.9035568396931356e-05,
+      "loss": 1.045,
+      "step": 1600
+    },
+    {
+      "epoch": 0.14570070832959742,
+      "grad_norm": 2.5223998582597855,
+      "learning_rate": 1.898575271495467e-05,
+      "loss": 1.0355,
+      "step": 1625
+    },
+    {
+      "epoch": 0.1479422576885143,
+      "grad_norm": 2.9220932656370366,
+      "learning_rate": 1.893593703297798e-05,
+      "loss": 1.0657,
+      "step": 1650
+    },
+    {
+      "epoch": 0.15018380704743117,
+      "grad_norm": 1.883244384210707,
+      "learning_rate": 1.8886121351001298e-05,
+      "loss": 1.0285,
+      "step": 1675
+    },
+    {
+      "epoch": 0.15242535640634808,
+      "grad_norm": 2.0400410969199445,
+      "learning_rate": 1.883630566902461e-05,
+      "loss": 1.0292,
+      "step": 1700
+    },
+    {
+      "epoch": 0.15466690576526496,
+      "grad_norm": 1.7753007869372996,
+      "learning_rate": 1.8786489987047924e-05,
+      "loss": 1.0508,
+      "step": 1725
+    },
+    {
+      "epoch": 0.15690845512418183,
+      "grad_norm": 1.6604310223265337,
+      "learning_rate": 1.8736674305071237e-05,
+      "loss": 1.0183,
+      "step": 1750
+    },
+    {
+      "epoch": 0.1591500044830987,
+      "grad_norm": 1.9310548345394576,
+      "learning_rate": 1.8686858623094554e-05,
+      "loss": 1.0438,
+      "step": 1775
+    },
+    {
+      "epoch": 0.1613915538420156,
+      "grad_norm": 1.8571948617085183,
+      "learning_rate": 1.8637042941117863e-05,
+      "loss": 1.0299,
+      "step": 1800
+    },
+    {
+      "epoch": 0.1636331032009325,
+      "grad_norm": 2.0999139482201037,
+      "learning_rate": 1.858722725914118e-05,
+      "loss": 1.0357,
+      "step": 1825
+    },
+    {
+      "epoch": 0.16587465255984937,
+      "grad_norm": 1.8302941023869517,
+      "learning_rate": 1.8537411577164493e-05,
+      "loss": 1.0394,
+      "step": 1850
+    },
+    {
+      "epoch": 0.16811620191876625,
+      "grad_norm": 1.7860535462449534,
+      "learning_rate": 1.8487595895187806e-05,
+      "loss": 1.0026,
+      "step": 1875
+    },
+    {
+      "epoch": 0.17035775127768313,
+      "grad_norm": 1.8983118114362876,
+      "learning_rate": 1.8437780213211122e-05,
+      "loss": 1.0178,
+      "step": 1900
+    },
+    {
+      "epoch": 0.1725993006366,
+      "grad_norm": 2.205679465328378,
+      "learning_rate": 1.8387964531234435e-05,
+      "loss": 1.0327,
+      "step": 1925
+    },
+    {
+      "epoch": 0.1748408499955169,
+      "grad_norm": 2.134664272100197,
+      "learning_rate": 1.8338148849257748e-05,
+      "loss": 1.029,
+      "step": 1950
+    },
+    {
+      "epoch": 0.1770823993544338,
+      "grad_norm": 1.9531002021208808,
+      "learning_rate": 1.828833316728106e-05,
+      "loss": 1.0521,
+      "step": 1975
+    },
+    {
+      "epoch": 0.17932394871335067,
+      "grad_norm": 1.9511745913401646,
+      "learning_rate": 1.8238517485304374e-05,
+      "loss": 1.0387,
+      "step": 2000
+    },
+    {
+      "epoch": 0.18156549807226754,
+      "grad_norm": 1.7617448914936356,
+      "learning_rate": 1.8188701803327687e-05,
+      "loss": 1.0312,
+      "step": 2025
+    },
+    {
+      "epoch": 0.18380704743118442,
+      "grad_norm": 1.8660119308426493,
+      "learning_rate": 1.8138886121351004e-05,
+      "loss": 1.001,
+      "step": 2050
+    },
+    {
+      "epoch": 0.18604859679010133,
+      "grad_norm": 11.361341672786956,
+      "learning_rate": 1.8089070439374317e-05,
+      "loss": 1.0323,
+      "step": 2075
+    },
+    {
+      "epoch": 0.1882901461490182,
+      "grad_norm": 1.9440541229934303,
+      "learning_rate": 1.803925475739763e-05,
+      "loss": 1.0292,
+      "step": 2100
+    },
+    {
+      "epoch": 0.19053169550793508,
+      "grad_norm": 1.6998031346610352,
+      "learning_rate": 1.7989439075420946e-05,
+      "loss": 1.0125,
+      "step": 2125
+    },
+    {
+      "epoch": 0.19277324486685196,
+      "grad_norm": 2.1474706821772123,
+      "learning_rate": 1.793962339344426e-05,
+      "loss": 0.9816,
+      "step": 2150
+    },
+    {
+      "epoch": 0.19501479422576884,
+      "grad_norm": 1.7163947976457987,
+      "learning_rate": 1.788980771146757e-05,
+      "loss": 1.0192,
+      "step": 2175
+    },
+    {
+      "epoch": 0.19725634358468574,
+      "grad_norm": 1.8892267392764226,
+      "learning_rate": 1.7839992029490885e-05,
+      "loss": 1.0056,
+      "step": 2200
+    },
+    {
+      "epoch": 0.19949789294360262,
+      "grad_norm": 2.0524457692263076,
+      "learning_rate": 1.77901763475142e-05,
+      "loss": 1.0103,
+      "step": 2225
+    },
+    {
+      "epoch": 0.2017394423025195,
+      "grad_norm": 1.8606062165245123,
+      "learning_rate": 1.774036066553751e-05,
+      "loss": 1.0093,
+      "step": 2250
+    },
+    {
+      "epoch": 0.20398099166143638,
+      "grad_norm": 1.6620327251666154,
+      "learning_rate": 1.7690544983560828e-05,
+      "loss": 1.0018,
+      "step": 2275
+    },
+    {
+      "epoch": 0.20622254102035326,
+      "grad_norm": 1.8581969734354233,
+      "learning_rate": 1.764072930158414e-05,
+      "loss": 0.9882,
+      "step": 2300
+    },
+    {
+      "epoch": 0.20846409037927016,
+      "grad_norm": 1.9329085125000098,
+      "learning_rate": 1.7590913619607454e-05,
+      "loss": 1.0029,
+      "step": 2325
+    },
+    {
+      "epoch": 0.21070563973818704,
+      "grad_norm": 1.9189341443783814,
+      "learning_rate": 1.7541097937630767e-05,
+      "loss": 0.9814,
+      "step": 2350
+    },
+    {
+      "epoch": 0.21294718909710392,
+      "grad_norm": 1.8381222846173,
+      "learning_rate": 1.749128225565408e-05,
+      "loss": 0.9953,
+      "step": 2375
+    },
+    {
+      "epoch": 0.2151887384560208,
+      "grad_norm": 1.8899248374704989,
+      "learning_rate": 1.7441466573677396e-05,
+      "loss": 0.9823,
+      "step": 2400
+    },
+    {
+      "epoch": 0.21743028781493767,
+      "grad_norm": 1.7449302218389742,
+      "learning_rate": 1.739165089170071e-05,
+      "loss": 0.9797,
+      "step": 2425
+    },
+    {
+      "epoch": 0.21967183717385458,
+      "grad_norm": 1.8919151945987245,
+      "learning_rate": 1.7341835209724022e-05,
+      "loss": 1.0136,
+      "step": 2450
+    },
+    {
+      "epoch": 0.22191338653277146,
+      "grad_norm": 1.7269209983774687,
+      "learning_rate": 1.7292019527747335e-05,
+      "loss": 0.9936,
+      "step": 2475
+    },
+    {
+      "epoch": 0.22415493589168833,
+      "grad_norm": 2.0801589205177073,
+      "learning_rate": 1.7242203845770652e-05,
+      "loss": 0.9806,
+      "step": 2500
+    },
+    {
+      "epoch": 0.2263964852506052,
+      "grad_norm": 1.9132953909491215,
+      "learning_rate": 1.719238816379396e-05,
+      "loss": 0.9773,
+      "step": 2525
+    },
+    {
+      "epoch": 0.2286380346095221,
+      "grad_norm": 2.121494652645993,
+      "learning_rate": 1.7142572481817278e-05,
+      "loss": 0.9913,
+      "step": 2550
+    },
+    {
+      "epoch": 0.230879583968439,
+      "grad_norm": 1.984427309513833,
+      "learning_rate": 1.709275679984059e-05,
+      "loss": 0.9777,
+      "step": 2575
+    },
+    {
+      "epoch": 0.23312113332735587,
+      "grad_norm": 1.8076604331872659,
+      "learning_rate": 1.7042941117863904e-05,
+      "loss": 0.9916,
+      "step": 2600
+    },
+    {
+      "epoch": 0.23536268268627275,
+      "grad_norm": 1.729667080715962,
+      "learning_rate": 1.699312543588722e-05,
+      "loss": 0.959,
+      "step": 2625
+    },
+    {
+      "epoch": 0.23760423204518963,
+      "grad_norm": 1.6461940121533178,
+      "learning_rate": 1.6943309753910533e-05,
+      "loss": 0.9356,
+      "step": 2650
+    },
+    {
+      "epoch": 0.2398457814041065,
+      "grad_norm": 1.7818239495803552,
+      "learning_rate": 1.6893494071933847e-05,
+      "loss": 0.9836,
+      "step": 2675
+    },
+    {
+      "epoch": 0.2420873307630234,
+      "grad_norm": 1.7098481517946613,
+      "learning_rate": 1.684367838995716e-05,
+      "loss": 0.9626,
+      "step": 2700
+    },
+    {
+      "epoch": 0.2443288801219403,
+      "grad_norm": 1.865561495461263,
+      "learning_rate": 1.6793862707980473e-05,
+      "loss": 0.9773,
+      "step": 2725
+    },
+    {
+      "epoch": 0.24657042948085717,
+      "grad_norm": 1.7896064708721888,
+      "learning_rate": 1.6744047026003786e-05,
+      "loss": 0.9819,
+      "step": 2750
+    },
+    {
+      "epoch": 0.24881197883977405,
+      "grad_norm": 1.7260499292304918,
+      "learning_rate": 1.6694231344027102e-05,
+      "loss": 0.9752,
+      "step": 2775
+    },
+    {
+      "epoch": 0.2510535281986909,
+      "grad_norm": 1.826190790491217,
+      "learning_rate": 1.6644415662050415e-05,
+      "loss": 0.9346,
+      "step": 2800
+    },
+    {
+      "epoch": 0.25329507755760783,
+      "grad_norm": 1.7454817242814047,
+      "learning_rate": 1.6594599980073728e-05,
+      "loss": 0.9609,
+      "step": 2825
+    },
+    {
+      "epoch": 0.2555366269165247,
+      "grad_norm": 1.7578596385064043,
+      "learning_rate": 1.6544784298097045e-05,
+      "loss": 0.9556,
+      "step": 2850
+    },
+    {
+      "epoch": 0.2577781762754416,
+      "grad_norm": 1.7683018696985564,
+      "learning_rate": 1.6494968616120358e-05,
+      "loss": 0.9484,
+      "step": 2875
+    },
+    {
+      "epoch": 0.2600197256343585,
+      "grad_norm": 1.7223372499585023,
+      "learning_rate": 1.6445152934143667e-05,
+      "loss": 0.968,
+      "step": 2900
+    },
+    {
+      "epoch": 0.26226127499327534,
+      "grad_norm": 1.808053969311946,
+      "learning_rate": 1.6395337252166984e-05,
+      "loss": 0.9536,
+      "step": 2925
+    },
+    {
+      "epoch": 0.26450282435219225,
+      "grad_norm": 1.9043794632911786,
+      "learning_rate": 1.6345521570190297e-05,
+      "loss": 0.9442,
+      "step": 2950
+    },
+    {
+      "epoch": 0.2667443737111091,
+      "grad_norm": 1.7895687686236537,
+      "learning_rate": 1.629570588821361e-05,
+      "loss": 0.9466,
+      "step": 2975
+    },
+    {
+      "epoch": 0.268985923070026,
+      "grad_norm": 1.934781759123613,
+      "learning_rate": 1.6245890206236926e-05,
+      "loss": 0.9807,
+      "step": 3000
+    },
+    {
+      "epoch": 0.2712274724289429,
+      "grad_norm": 1.632988026978096,
+      "learning_rate": 1.619607452426024e-05,
+      "loss": 0.9594,
+      "step": 3025
+    },
+    {
+      "epoch": 0.27346902178785976,
+      "grad_norm": 2.027393243768135,
+      "learning_rate": 1.6146258842283552e-05,
+      "loss": 0.9531,
+      "step": 3050
+    },
+    {
+      "epoch": 0.27571057114677666,
+      "grad_norm": 1.8992087782858946,
+      "learning_rate": 1.6096443160306865e-05,
+      "loss": 0.9437,
+      "step": 3075
+    },
+    {
+      "epoch": 0.2779521205056935,
+      "grad_norm": 1.7343465694135203,
+      "learning_rate": 1.6046627478330178e-05,
+      "loss": 0.9626,
+      "step": 3100
+    },
+    {
+      "epoch": 0.2801936698646104,
+      "grad_norm": 1.7072938434556733,
+      "learning_rate": 1.5996811796353495e-05,
+      "loss": 0.9453,
+      "step": 3125
+    },
+    {
+      "epoch": 0.2824352192235273,
+      "grad_norm": 1.8218156899532207,
+      "learning_rate": 1.5946996114376808e-05,
+      "loss": 0.9304,
+      "step": 3150
+    },
+    {
+      "epoch": 0.2846767685824442,
+      "grad_norm": 1.6937676571633637,
+      "learning_rate": 1.589718043240012e-05,
+      "loss": 0.9447,
+      "step": 3175
+    },
+    {
+      "epoch": 0.2869183179413611,
+      "grad_norm": 1.644893250819176,
+      "learning_rate": 1.5847364750423434e-05,
+      "loss": 0.9503,
+      "step": 3200
+    },
+    {
+      "epoch": 0.28915986730027793,
+      "grad_norm": 1.8909556942793773,
+      "learning_rate": 1.579754906844675e-05,
+      "loss": 0.949,
+      "step": 3225
+    },
+    {
+      "epoch": 0.29140141665919483,
+      "grad_norm": 1.8529318404284596,
+      "learning_rate": 1.574773338647006e-05,
+      "loss": 0.9152,
+      "step": 3250
+    },
+    {
+      "epoch": 0.29364296601811174,
+      "grad_norm": 1.7004297471160559,
+      "learning_rate": 1.5697917704493376e-05,
+      "loss": 0.9112,
+      "step": 3275
+    },
+    {
+      "epoch": 0.2958845153770286,
+      "grad_norm": 1.9502983060858345,
+      "learning_rate": 1.564810202251669e-05,
+      "loss": 0.9569,
+      "step": 3300
+    },
+    {
+      "epoch": 0.2981260647359455,
+      "grad_norm": 1.7459285738352261,
+      "learning_rate": 1.5598286340540002e-05,
+      "loss": 0.9441,
+      "step": 3325
+    },
+    {
+      "epoch": 0.30036761409486235,
+      "grad_norm": 1.6839416479204592,
+      "learning_rate": 1.554847065856332e-05,
+      "loss": 0.9343,
+      "step": 3350
+    },
+    {
+      "epoch": 0.30260916345377925,
+      "grad_norm": 1.7972168152482286,
+      "learning_rate": 1.5498654976586632e-05,
+      "loss": 0.93,
+      "step": 3375
+    },
+    {
+      "epoch": 0.30485071281269616,
+      "grad_norm": 1.7165679877234483,
+      "learning_rate": 1.5448839294609945e-05,
+      "loss": 0.9024,
+      "step": 3400
+    },
+    {
+      "epoch": 0.307092262171613,
+      "grad_norm": 1.8154734324573523,
+      "learning_rate": 1.5399023612633258e-05,
+      "loss": 0.9301,
+      "step": 3425
+    },
+    {
+      "epoch": 0.3093338115305299,
+      "grad_norm": 1.6952336101878391,
+      "learning_rate": 1.534920793065657e-05,
+      "loss": 0.9204,
+      "step": 3450
+    },
+    {
+      "epoch": 0.31157536088944676,
+      "grad_norm": 1.7430417778073755,
+      "learning_rate": 1.5299392248679884e-05,
+      "loss": 0.9325,
+      "step": 3475
+    },
+    {
+      "epoch": 0.31381691024836367,
+      "grad_norm": 1.8264268621395723,
+      "learning_rate": 1.5249576566703199e-05,
+      "loss": 0.9388,
+      "step": 3500
+    },
+    {
+      "epoch": 0.3160584596072806,
+      "grad_norm": 1.7279147424934178,
+      "learning_rate": 1.5199760884726513e-05,
+      "loss": 0.9532,
+      "step": 3525
+    },
+    {
+      "epoch": 0.3183000089661974,
+      "grad_norm": 1.7829003359104079,
+      "learning_rate": 1.5149945202749826e-05,
+      "loss": 0.9156,
+      "step": 3550
+    },
+    {
+      "epoch": 0.32054155832511433,
+      "grad_norm": 1.8115804249304372,
+      "learning_rate": 1.5100129520773141e-05,
+      "loss": 0.9126,
+      "step": 3575
+    },
+    {
+      "epoch": 0.3227831076840312,
+      "grad_norm": 1.8101472164090928,
+      "learning_rate": 1.5050313838796456e-05,
+      "loss": 0.9276,
+      "step": 3600
+    },
+    {
+      "epoch": 0.3250246570429481,
+      "grad_norm": 1.9070110243849654,
+      "learning_rate": 1.5000498156819767e-05,
+      "loss": 0.9128,
+      "step": 3625
+    },
+    {
+      "epoch": 0.327266206401865,
+      "grad_norm": 1.9072311557403854,
+      "learning_rate": 1.4950682474843082e-05,
+      "loss": 0.9184,
+      "step": 3650
+    },
+    {
+      "epoch": 0.32950775576078184,
+      "grad_norm": 1.61923199587523,
+      "learning_rate": 1.4900866792866395e-05,
+      "loss": 0.911,
+      "step": 3675
+    },
+    {
+      "epoch": 0.33174930511969875,
+      "grad_norm": 1.8120353797625837,
+      "learning_rate": 1.485105111088971e-05,
+      "loss": 0.915,
+      "step": 3700
+    },
+    {
+      "epoch": 0.3339908544786156,
+      "grad_norm": 1.7354507845151927,
+      "learning_rate": 1.4801235428913023e-05,
+      "loss": 0.9115,
+      "step": 3725
+    },
+    {
+      "epoch": 0.3362324038375325,
+      "grad_norm": 1.7796643660754665,
+      "learning_rate": 1.4751419746936338e-05,
+      "loss": 0.9145,
+      "step": 3750
+    },
+    {
+      "epoch": 0.3384739531964494,
+      "grad_norm": 1.855968151723882,
+      "learning_rate": 1.4701604064959652e-05,
+      "loss": 0.9056,
+      "step": 3775
+    },
+    {
+      "epoch": 0.34071550255536626,
+      "grad_norm": 1.9082686727020328,
+      "learning_rate": 1.4651788382982964e-05,
+      "loss": 0.9304,
+      "step": 3800
+    },
+    {
+      "epoch": 0.34295705191428316,
+      "grad_norm": 1.8449390404041304,
+      "learning_rate": 1.4601972701006277e-05,
+      "loss": 0.8985,
+      "step": 3825
+    },
+    {
+      "epoch": 0.3451986012732,
+      "grad_norm": 1.7578794796486368,
+      "learning_rate": 1.4552157019029591e-05,
+      "loss": 0.8969,
+      "step": 3850
+    },
+    {
+      "epoch": 0.3474401506321169,
+      "grad_norm": 1.8367706573696312,
+      "learning_rate": 1.4502341337052906e-05,
+      "loss": 0.915,
+      "step": 3875
+    },
+    {
+      "epoch": 0.3496816999910338,
+      "grad_norm": 1.8822503403709299,
+      "learning_rate": 1.4452525655076219e-05,
+      "loss": 0.914,
+      "step": 3900
+    },
+    {
+      "epoch": 0.3519232493499507,
+      "grad_norm": 1.7324094016304221,
+      "learning_rate": 1.4402709973099534e-05,
+      "loss": 0.9103,
+      "step": 3925
+    },
+    {
+      "epoch": 0.3541647987088676,
+      "grad_norm": 1.924493271883657,
+      "learning_rate": 1.4352894291122847e-05,
+      "loss": 0.8822,
+      "step": 3950
+    },
+    {
+      "epoch": 0.35640634806778443,
+      "grad_norm": 1.7603655512407765,
+      "learning_rate": 1.430307860914616e-05,
+      "loss": 0.9047,
+      "step": 3975
+    },
+    {
+      "epoch": 0.35864789742670133,
+      "grad_norm": 1.728593406803126,
+      "learning_rate": 1.4253262927169473e-05,
+      "loss": 0.9034,
+      "step": 4000
+    },
+    {
+      "epoch": 0.36088944678561824,
+      "grad_norm": 1.7079384156065074,
+      "learning_rate": 1.4203447245192788e-05,
+      "loss": 0.9108,
+      "step": 4025
+    },
+    {
+      "epoch": 0.3631309961445351,
+      "grad_norm": 1.8124067484354314,
+      "learning_rate": 1.41536315632161e-05,
+      "loss": 0.8731,
+      "step": 4050
+    },
+    {
+      "epoch": 0.365372545503452,
+      "grad_norm": 1.6392410836376878,
+      "learning_rate": 1.4103815881239415e-05,
+      "loss": 0.8961,
+      "step": 4075
+    },
+    {
+      "epoch": 0.36761409486236885,
+      "grad_norm": 1.8375262874030192,
+      "learning_rate": 1.405400019926273e-05,
+      "loss": 0.9039,
+      "step": 4100
+    },
+    {
+      "epoch": 0.36985564422128575,
+      "grad_norm": 1.7237334097670618,
+      "learning_rate": 1.4004184517286043e-05,
+      "loss": 0.9017,
+      "step": 4125
+    },
+    {
+      "epoch": 0.37209719358020266,
+      "grad_norm": 1.7732172478528363,
+      "learning_rate": 1.3954368835309358e-05,
+      "loss": 0.8707,
+      "step": 4150
+    },
+    {
+      "epoch": 0.3743387429391195,
+      "grad_norm": 1.798174483175535,
+      "learning_rate": 1.390455315333267e-05,
+      "loss": 0.8951,
+      "step": 4175
+    },
+    {
+      "epoch": 0.3765802922980364,
+      "grad_norm": 1.6403444255783957,
+      "learning_rate": 1.3854737471355984e-05,
+      "loss": 0.8867,
+      "step": 4200
+    },
+    {
+      "epoch": 0.37882184165695326,
+      "grad_norm": 1.630947223766077,
+      "learning_rate": 1.3804921789379297e-05,
+      "loss": 0.8633,
+      "step": 4225
+    },
+    {
+      "epoch": 0.38106339101587017,
+      "grad_norm": 1.7170897564750465,
+      "learning_rate": 1.3755106107402612e-05,
+      "loss": 0.8984,
+      "step": 4250
+    },
+    {
+      "epoch": 0.3833049403747871,
+      "grad_norm": 2.375355817440413,
+      "learning_rate": 1.3705290425425925e-05,
+      "loss": 0.9036,
+      "step": 4275
+    },
+    {
+      "epoch": 0.3855464897337039,
+      "grad_norm": 1.773676455707909,
+      "learning_rate": 1.365547474344924e-05,
+      "loss": 0.8859,
+      "step": 4300
+    },
+    {
+      "epoch": 0.38778803909262083,
+      "grad_norm": 2.2923198536180385,
+      "learning_rate": 1.3605659061472554e-05,
+      "loss": 0.8914,
+      "step": 4325
+    },
+    {
+      "epoch": 0.3900295884515377,
+      "grad_norm": 1.5848618729646098,
+      "learning_rate": 1.3555843379495866e-05,
+      "loss": 0.8832,
+      "step": 4350
+    },
+    {
+      "epoch": 0.3922711378104546,
+      "grad_norm": 1.8306249640013397,
+      "learning_rate": 1.350602769751918e-05,
+      "loss": 0.8689,
+      "step": 4375
+    },
+    {
+      "epoch": 0.3945126871693715,
+      "grad_norm": 1.7205681208039685,
+      "learning_rate": 1.3456212015542493e-05,
+      "loss": 0.8886,
+      "step": 4400
+    },
+    {
+      "epoch": 0.39675423652828834,
+      "grad_norm": 1.7563808610676828,
+      "learning_rate": 1.3406396333565808e-05,
+      "loss": 0.8957,
+      "step": 4425
+    },
+    {
+      "epoch": 0.39899578588720525,
+      "grad_norm": 1.762473526098044,
+      "learning_rate": 1.3356580651589121e-05,
+      "loss": 0.886,
+      "step": 4450
+    },
+    {
+      "epoch": 0.4012373352461221,
+      "grad_norm": 1.5235508352345122,
+      "learning_rate": 1.3306764969612436e-05,
+      "loss": 0.9023,
+      "step": 4475
+    },
+    {
+      "epoch": 0.403478884605039,
+      "grad_norm": 1.6080245871591732,
+      "learning_rate": 1.325694928763575e-05,
+      "loss": 0.8742,
+      "step": 4500
+    },
+    {
+      "epoch": 0.4057204339639559,
+      "grad_norm": 1.6042016149275076,
+      "learning_rate": 1.3207133605659062e-05,
+      "loss": 0.8861,
+      "step": 4525
+    },
+    {
+      "epoch": 0.40796198332287276,
+      "grad_norm": 1.833739247341173,
+      "learning_rate": 1.3157317923682375e-05,
+      "loss": 0.8808,
+      "step": 4550
+    },
+    {
+      "epoch": 0.41020353268178966,
+      "grad_norm": 1.6633181057260342,
+      "learning_rate": 1.310750224170569e-05,
+      "loss": 0.8519,
+      "step": 4575
+    },
+    {
+      "epoch": 0.4124450820407065,
+      "grad_norm": 1.77817726319169,
+      "learning_rate": 1.3057686559729004e-05,
+      "loss": 0.8623,
+      "step": 4600
+    },
+    {
+      "epoch": 0.4146866313996234,
+      "grad_norm": 1.9820958145272036,
+      "learning_rate": 1.3007870877752317e-05,
+      "loss": 0.8659,
+      "step": 4625
+    },
+    {
+      "epoch": 0.4169281807585403,
+      "grad_norm": 1.6475751439718662,
+      "learning_rate": 1.2958055195775632e-05,
+      "loss": 0.8863,
+      "step": 4650
+    },
+    {
+      "epoch": 0.4191697301174572,
+      "grad_norm": 1.6563594872670169,
+      "learning_rate": 1.2908239513798945e-05,
+      "loss": 0.852,
+      "step": 4675
+    },
+    {
+      "epoch": 0.4214112794763741,
+      "grad_norm": 1.7048605228423215,
+      "learning_rate": 1.2858423831822258e-05,
+      "loss": 0.8649,
+      "step": 4700
+    },
+    {
+      "epoch": 0.42365282883529093,
+      "grad_norm": 1.625027526065513,
+      "learning_rate": 1.2808608149845571e-05,
+      "loss": 0.8658,
+      "step": 4725
+    },
+    {
+      "epoch": 0.42589437819420783,
+      "grad_norm": 1.918671896492615,
+      "learning_rate": 1.2758792467868886e-05,
+      "loss": 0.8478,
+      "step": 4750
+    },
+    {
+      "epoch": 0.42813592755312474,
+      "grad_norm": 1.7529097573536403,
+      "learning_rate": 1.2708976785892199e-05,
+      "loss": 0.8863,
+      "step": 4775
+    },
+    {
+      "epoch": 0.4303774769120416,
+      "grad_norm": 1.7580856057956353,
+      "learning_rate": 1.2659161103915514e-05,
+      "loss": 0.8543,
+      "step": 4800
+    },
+    {
+      "epoch": 0.4326190262709585,
+      "grad_norm": 1.7264754733728158,
+      "learning_rate": 1.2609345421938828e-05,
+      "loss": 0.88,
+      "step": 4825
+    },
+    {
+      "epoch": 0.43486057562987535,
+      "grad_norm": 1.6664400318355115,
+      "learning_rate": 1.2559529739962142e-05,
+      "loss": 0.865,
+      "step": 4850
+    },
+    {
+      "epoch": 0.43710212498879225,
+      "grad_norm": 1.5810013518562152,
+      "learning_rate": 1.2509714057985456e-05,
+      "loss": 0.8644,
+      "step": 4875
+    },
+    {
+      "epoch": 0.43934367434770916,
+      "grad_norm": 1.7637476166127988,
+      "learning_rate": 1.2459898376008768e-05,
+      "loss": 0.845,
+      "step": 4900
+    },
+    {
+      "epoch": 0.441585223706626,
+      "grad_norm": 1.7932279049924675,
+      "learning_rate": 1.2410082694032082e-05,
+      "loss": 0.8651,
+      "step": 4925
+    },
+    {
+      "epoch": 0.4438267730655429,
+      "grad_norm": 1.829561755443512,
+      "learning_rate": 1.2360267012055395e-05,
+      "loss": 0.8504,
+      "step": 4950
+    },
+    {
+      "epoch": 0.44606832242445976,
+      "grad_norm": 1.6188361315894273,
+      "learning_rate": 1.231045133007871e-05,
+      "loss": 0.8161,
+      "step": 4975
+    },
+    {
+      "epoch": 0.44830987178337667,
+      "grad_norm": 1.7056183046742297,
+      "learning_rate": 1.2260635648102023e-05,
+      "loss": 0.8591,
+      "step": 5000
+    },
+    {
+      "epoch": 0.4505514211422936,
+      "grad_norm": 1.8841012895490568,
+      "learning_rate": 1.2210819966125338e-05,
+      "loss": 0.861,
+      "step": 5025
+    },
+    {
+      "epoch": 0.4527929705012104,
+      "grad_norm": 1.6238068299735378,
+      "learning_rate": 1.2161004284148653e-05,
+      "loss": 0.8575,
+      "step": 5050
+    },
+    {
+      "epoch": 0.45503451986012733,
+      "grad_norm": 1.8397094786253432,
+      "learning_rate": 1.2111188602171964e-05,
+      "loss": 0.8822,
+      "step": 5075
+    },
+    {
+      "epoch": 0.4572760692190442,
+      "grad_norm": 1.864241679602278,
+      "learning_rate": 1.2061372920195279e-05,
+      "loss": 0.8706,
+      "step": 5100
+    },
+    {
+      "epoch": 0.4595176185779611,
+      "grad_norm": 1.961344695482636,
+      "learning_rate": 1.2011557238218592e-05,
+      "loss": 0.8465,
+      "step": 5125
+    },
+    {
+      "epoch": 0.461759167936878,
+      "grad_norm": 1.7654809958600572,
+      "learning_rate": 1.1961741556241906e-05,
+      "loss": 0.8744,
+      "step": 5150
+    },
+    {
+      "epoch": 0.46400071729579484,
+      "grad_norm": 1.7238964433167303,
+      "learning_rate": 1.191192587426522e-05,
+      "loss": 0.8475,
+      "step": 5175
+    },
+    {
+      "epoch": 0.46624226665471175,
+      "grad_norm": 1.7695094943018663,
+      "learning_rate": 1.1862110192288534e-05,
+      "loss": 0.8193,
+      "step": 5200
+    },
+    {
+      "epoch": 0.4684838160136286,
+      "grad_norm": 1.8313450949005072,
+      "learning_rate": 1.1812294510311847e-05,
+      "loss": 0.8543,
+      "step": 5225
+    },
+    {
+      "epoch": 0.4707253653725455,
+      "grad_norm": 1.7242055485237682,
+      "learning_rate": 1.176247882833516e-05,
+      "loss": 0.8524,
+      "step": 5250
+    },
+    {
+      "epoch": 0.4729669147314624,
+      "grad_norm": 1.685334541109067,
+      "learning_rate": 1.1712663146358473e-05,
+      "loss": 0.8437,
+      "step": 5275
+    },
+    {
+      "epoch": 0.47520846409037926,
+      "grad_norm": 1.7482758279879385,
+      "learning_rate": 1.1662847464381788e-05,
+      "loss": 0.8305,
+      "step": 5300
+    },
+    {
+      "epoch": 0.47745001344929616,
+      "grad_norm": 3.4351483388103388,
+      "learning_rate": 1.1613031782405103e-05,
+      "loss": 0.853,
+      "step": 5325
+    },
+    {
+      "epoch": 0.479691562808213,
+      "grad_norm": 1.6345096360692302,
+      "learning_rate": 1.1563216100428416e-05,
+      "loss": 0.8328,
+      "step": 5350
+    },
+    {
+      "epoch": 0.4819331121671299,
+      "grad_norm": 1.8613988702465172,
+      "learning_rate": 1.151340041845173e-05,
+      "loss": 0.8475,
+      "step": 5375
+    },
+    {
+      "epoch": 0.4841746615260468,
+      "grad_norm": 1.7381762438125414,
+      "learning_rate": 1.1463584736475044e-05,
+      "loss": 0.8366,
+      "step": 5400
+    },
+    {
+      "epoch": 0.4864162108849637,
+      "grad_norm": 1.9220405850041657,
+      "learning_rate": 1.1413769054498357e-05,
+      "loss": 0.832,
+      "step": 5425
+    },
+    {
+      "epoch": 0.4886577602438806,
+      "grad_norm": 1.7762301237994864,
+      "learning_rate": 1.136395337252167e-05,
+      "loss": 0.8288,
+      "step": 5450
+    },
+    {
+      "epoch": 0.49089930960279743,
+      "grad_norm": 1.6336059312280666,
+      "learning_rate": 1.1314137690544984e-05,
+      "loss": 0.8428,
+      "step": 5475
+    },
+    {
+      "epoch": 0.49314085896171433,
+      "grad_norm": 1.7113743327911453,
+      "learning_rate": 1.1264322008568297e-05,
+      "loss": 0.8245,
+      "step": 5500
+    },
+    {
+      "epoch": 0.49538240832063124,
+      "grad_norm": 1.6955896520800513,
+      "learning_rate": 1.1214506326591612e-05,
+      "loss": 0.854,
+      "step": 5525
+    },
+    {
+      "epoch": 0.4976239576795481,
+      "grad_norm": 1.745123834536872,
+      "learning_rate": 1.1164690644614927e-05,
+      "loss": 0.8407,
+      "step": 5550
+    },
+    {
+      "epoch": 0.499865507038465,
+      "grad_norm": 1.7031285490401316,
+      "learning_rate": 1.111487496263824e-05,
+      "loss": 0.8407,
+      "step": 5575
+    },
+    {
+      "epoch": 0.5021070563973818,
+      "grad_norm": 1.6770089078383563,
+      "learning_rate": 1.1065059280661555e-05,
+      "loss": 0.8201,
+      "step": 5600
+    },
+    {
+      "epoch": 0.5043486057562988,
+      "grad_norm": 1.7307655888153608,
+      "learning_rate": 1.1015243598684866e-05,
+      "loss": 0.8338,
+      "step": 5625
+    },
+    {
+      "epoch": 0.5065901551152157,
+      "grad_norm": 1.8528909963261024,
+      "learning_rate": 1.096542791670818e-05,
+      "loss": 0.8307,
+      "step": 5650
+    },
+    {
+      "epoch": 0.5088317044741325,
+      "grad_norm": 1.8698755576150798,
+      "learning_rate": 1.0915612234731494e-05,
+      "loss": 0.8448,
+      "step": 5675
+    },
+    {
+      "epoch": 0.5110732538330494,
+      "grad_norm": 1.80016651013364,
+      "learning_rate": 1.0865796552754808e-05,
+      "loss": 0.8335,
+      "step": 5700
+    },
+    {
+      "epoch": 0.5133148031919663,
+      "grad_norm": 1.6573575321981893,
+      "learning_rate": 1.0815980870778121e-05,
+      "loss": 0.8406,
+      "step": 5725
+    },
+    {
+      "epoch": 0.5155563525508832,
+      "grad_norm": 1.8492211722078626,
+      "learning_rate": 1.0766165188801436e-05,
+      "loss": 0.8172,
+      "step": 5750
+    },
+    {
+      "epoch": 0.5177979019098,
+      "grad_norm": 1.6035391080567873,
+      "learning_rate": 1.0716349506824751e-05,
+      "loss": 0.8392,
+      "step": 5775
+    },
+    {
+      "epoch": 0.520039451268717,
+      "grad_norm": 1.8244861205863412,
+      "learning_rate": 1.0666533824848062e-05,
+      "loss": 0.8255,
+      "step": 5800
+    },
+    {
+      "epoch": 0.5222810006276338,
+      "grad_norm": 1.9845349836801902,
+      "learning_rate": 1.0616718142871377e-05,
+      "loss": 0.7773,
+      "step": 5825
+    },
+    {
+      "epoch": 0.5245225499865507,
+      "grad_norm": 1.6692235658845165,
+      "learning_rate": 1.056690246089469e-05,
+      "loss": 0.8204,
+      "step": 5850
+    },
+    {
+      "epoch": 0.5267640993454676,
+      "grad_norm": 1.77718790971494,
+      "learning_rate": 1.0517086778918005e-05,
+      "loss": 0.838,
+      "step": 5875
+    },
+    {
+      "epoch": 0.5290056487043845,
+      "grad_norm": 2.216535651554211,
+      "learning_rate": 1.0467271096941318e-05,
+      "loss": 0.8032,
+      "step": 5900
+    },
+    {
+      "epoch": 0.5312471980633013,
+      "grad_norm": 1.51672462878331,
+      "learning_rate": 1.0417455414964633e-05,
+      "loss": 0.8134,
+      "step": 5925
+    },
+    {
+      "epoch": 0.5334887474222182,
+      "grad_norm": 1.5842154935520327,
+      "learning_rate": 1.0367639732987946e-05,
+      "loss": 0.8156,
+      "step": 5950
+    },
+    {
+      "epoch": 0.5357302967811352,
+      "grad_norm": 1.8084527777329682,
+      "learning_rate": 1.0317824051011259e-05,
+      "loss": 0.8044,
+      "step": 5975
+    },
+    {
+      "epoch": 0.537971846140052,
+      "grad_norm": 1.8279664094537542,
+      "learning_rate": 1.0268008369034572e-05,
+      "loss": 0.8221,
+      "step": 6000
+    },
+    {
+      "epoch": 0.5402133954989689,
+      "grad_norm": 1.8246484550571145,
+      "learning_rate": 1.0218192687057886e-05,
+      "loss": 0.8113,
+      "step": 6025
+    },
+    {
+      "epoch": 0.5424549448578858,
+      "grad_norm": 1.9794225799144154,
+      "learning_rate": 1.0168377005081201e-05,
+      "loss": 0.8297,
+      "step": 6050
+    },
+    {
+      "epoch": 0.5446964942168027,
+      "grad_norm": 1.8107096798923765,
+      "learning_rate": 1.0118561323104514e-05,
+      "loss": 0.7963,
+      "step": 6075
+    },
+    {
+      "epoch": 0.5469380435757195,
+      "grad_norm": 1.7381439602233606,
+      "learning_rate": 1.0068745641127829e-05,
+      "loss": 0.8292,
+      "step": 6100
+    },
+    {
+      "epoch": 0.5491795929346365,
+      "grad_norm": 1.7511199022521768,
+      "learning_rate": 1.0018929959151142e-05,
+      "loss": 0.7881,
+      "step": 6125
+    },
+    {
+      "epoch": 0.5514211422935533,
+      "grad_norm": 1.8540296738093,
+      "learning_rate": 9.969114277174455e-06,
+      "loss": 0.819,
+      "step": 6150
+    },
+    {
+      "epoch": 0.5536626916524702,
+      "grad_norm": 1.604616094415389,
+      "learning_rate": 9.91929859519777e-06,
+      "loss": 0.8149,
+      "step": 6175
+    },
+    {
+      "epoch": 0.555904241011387,
+      "grad_norm": 1.7889735212307019,
+      "learning_rate": 9.869482913221083e-06,
+      "loss": 0.8174,
+      "step": 6200
+    },
+    {
+      "epoch": 0.558145790370304,
+      "grad_norm": 1.7124938531820217,
+      "learning_rate": 9.819667231244396e-06,
+      "loss": 0.8282,
+      "step": 6225
+    },
+    {
+      "epoch": 0.5603873397292208,
+      "grad_norm": 1.7697417082932925,
+      "learning_rate": 9.76985154926771e-06,
+      "loss": 0.8112,
+      "step": 6250
+    },
+    {
+      "epoch": 0.5626288890881377,
+      "grad_norm": 1.7720332425978838,
+      "learning_rate": 9.720035867291025e-06,
+      "loss": 0.8256,
+      "step": 6275
+    },
+    {
+      "epoch": 0.5648704384470546,
+      "grad_norm": 1.7791860279019227,
+      "learning_rate": 9.670220185314338e-06,
+      "loss": 0.7993,
+      "step": 6300
+    },
+    {
+      "epoch": 0.5671119878059715,
+      "grad_norm": 1.8298443186416975,
+      "learning_rate": 9.620404503337651e-06,
+      "loss": 0.8074,
+      "step": 6325
+    },
+    {
+      "epoch": 0.5693535371648883,
+      "grad_norm": 1.740618137225464,
+      "learning_rate": 9.570588821360966e-06,
+      "loss": 0.7915,
+      "step": 6350
+    },
+    {
+      "epoch": 0.5715950865238053,
+      "grad_norm": 1.6575823203829687,
+      "learning_rate": 9.520773139384279e-06,
+      "loss": 0.8061,
+      "step": 6375
+    },
+    {
+      "epoch": 0.5738366358827222,
+      "grad_norm": 1.594956412545229,
+      "learning_rate": 9.470957457407592e-06,
+      "loss": 0.7835,
+      "step": 6400
+    },
+    {
+      "epoch": 0.576078185241639,
+      "grad_norm": 1.7061113242845711,
+      "learning_rate": 9.421141775430907e-06,
+      "loss": 0.7786,
+      "step": 6425
+    },
+    {
+      "epoch": 0.5783197346005559,
+      "grad_norm": 1.7037561820889011,
+      "learning_rate": 9.37132609345422e-06,
+      "loss": 0.7847,
+      "step": 6450
+    },
+    {
+      "epoch": 0.5805612839594728,
+      "grad_norm": 1.6215842870197699,
+      "learning_rate": 9.321510411477533e-06,
+      "loss": 0.8216,
+      "step": 6475
+    },
+    {
+      "epoch": 0.5828028333183897,
+      "grad_norm": 1.7215498826410882,
+      "learning_rate": 9.271694729500848e-06,
+      "loss": 0.7908,
+      "step": 6500
+    },
+    {
+      "epoch": 0.5850443826773065,
+      "grad_norm": 1.6190690905298566,
+      "learning_rate": 9.221879047524162e-06,
+      "loss": 0.7975,
+      "step": 6525
+    },
+    {
+      "epoch": 0.5872859320362235,
+      "grad_norm": 1.7359669159042928,
+      "learning_rate": 9.172063365547475e-06,
+      "loss": 0.8038,
+      "step": 6550
+    },
+    {
+      "epoch": 0.5895274813951403,
+      "grad_norm": 1.6569209888458083,
+      "learning_rate": 9.122247683570788e-06,
+      "loss": 0.7865,
+      "step": 6575
+    },
+    {
+      "epoch": 0.5917690307540572,
+      "grad_norm": 1.7468326887043915,
+      "learning_rate": 9.072432001594103e-06,
+      "loss": 0.7945,
+      "step": 6600
+    },
+    {
+      "epoch": 0.5940105801129741,
+      "grad_norm": 1.89027859669638,
+      "learning_rate": 9.022616319617416e-06,
+      "loss": 0.7945,
+      "step": 6625
+    },
+    {
+      "epoch": 0.596252129471891,
+      "grad_norm": 3.1956433500954846,
+      "learning_rate": 8.972800637640729e-06,
+      "loss": 0.7832,
+      "step": 6650
+    },
+    {
+      "epoch": 0.5984936788308078,
+      "grad_norm": 1.6708308695139495,
+      "learning_rate": 8.922984955664044e-06,
+      "loss": 0.7804,
+      "step": 6675
+    },
+    {
+      "epoch": 0.6007352281897247,
+      "grad_norm": 1.6619509622184405,
+      "learning_rate": 8.873169273687357e-06,
+      "loss": 0.7736,
+      "step": 6700
+    },
+    {
+      "epoch": 0.6029767775486417,
+      "grad_norm": 1.703044857731027,
+      "learning_rate": 8.823353591710672e-06,
+      "loss": 0.7803,
+      "step": 6725
+    },
+    {
+      "epoch": 0.6052183269075585,
+      "grad_norm": 1.690170724872845,
+      "learning_rate": 8.773537909733985e-06,
+      "loss": 0.7739,
+      "step": 6750
+    },
+    {
+      "epoch": 0.6074598762664754,
+      "grad_norm": 1.792403817902017,
+      "learning_rate": 8.7237222277573e-06,
+      "loss": 0.8006,
+      "step": 6775
+    },
+    {
+      "epoch": 0.6097014256253923,
+      "grad_norm": 1.6433093780482904,
+      "learning_rate": 8.673906545780612e-06,
+      "loss": 0.7867,
+      "step": 6800
+    },
+    {
+      "epoch": 0.6119429749843092,
+      "grad_norm": 1.790404324898307,
+      "learning_rate": 8.624090863803927e-06,
+      "loss": 0.7849,
+      "step": 6825
+    },
+    {
+      "epoch": 0.614184524343226,
+      "grad_norm": 1.6454337492340885,
+      "learning_rate": 8.57427518182724e-06,
+      "loss": 0.7879,
+      "step": 6850
+    },
+    {
+      "epoch": 0.616426073702143,
+      "grad_norm": 1.7355887161904928,
+      "learning_rate": 8.524459499850553e-06,
+      "loss": 0.7768,
+      "step": 6875
+    },
+    {
+      "epoch": 0.6186676230610598,
+      "grad_norm": 1.7882226224444007,
+      "learning_rate": 8.474643817873868e-06,
+      "loss": 0.7994,
+      "step": 6900
+    },
+    {
+      "epoch": 0.6209091724199767,
+      "grad_norm": 1.6311627761596097,
+      "learning_rate": 8.424828135897181e-06,
+      "loss": 0.778,
+      "step": 6925
+    },
+    {
+      "epoch": 0.6231507217788935,
+      "grad_norm": 1.8193761919911795,
+      "learning_rate": 8.375012453920494e-06,
+      "loss": 0.8019,
+      "step": 6950
+    },
+    {
+      "epoch": 0.6253922711378105,
+      "grad_norm": 1.837184984077742,
+      "learning_rate": 8.325196771943809e-06,
+      "loss": 0.788,
+      "step": 6975
+    },
+    {
+      "epoch": 0.6276338204967273,
+      "grad_norm": 1.7295194630868602,
+      "learning_rate": 8.275381089967123e-06,
+      "loss": 0.79,
+      "step": 7000
+    },
+    {
+      "epoch": 0.6298753698556442,
+      "grad_norm": 1.7138938284046155,
+      "learning_rate": 8.225565407990437e-06,
+      "loss": 0.8014,
+      "step": 7025
+    },
+    {
+      "epoch": 0.6321169192145611,
+      "grad_norm": 1.761547883189452,
+      "learning_rate": 8.17574972601375e-06,
+      "loss": 0.7667,
+      "step": 7050
+    },
+    {
+      "epoch": 0.634358468573478,
+      "grad_norm": 1.733008722282232,
+      "learning_rate": 8.125934044037064e-06,
+      "loss": 0.7932,
+      "step": 7075
+    },
+    {
+      "epoch": 0.6366000179323948,
+      "grad_norm": 4.166995261072046,
+      "learning_rate": 8.076118362060377e-06,
+      "loss": 0.8139,
+      "step": 7100
+    },
+    {
+      "epoch": 0.6388415672913118,
+      "grad_norm": 2.775575951767444,
+      "learning_rate": 8.02630268008369e-06,
+      "loss": 0.8398,
+      "step": 7125
+    },
+    {
+      "epoch": 0.6410831166502287,
+      "grad_norm": 1.866468516722052,
+      "learning_rate": 7.976486998107005e-06,
+      "loss": 0.7846,
+      "step": 7150
+    },
+    {
+      "epoch": 0.6433246660091455,
+      "grad_norm": 1.7569056138933221,
+      "learning_rate": 7.926671316130318e-06,
+      "loss": 0.7579,
+      "step": 7175
+    },
+    {
+      "epoch": 0.6455662153680624,
+      "grad_norm": 1.7346922018694015,
+      "learning_rate": 7.876855634153631e-06,
+      "loss": 0.7668,
+      "step": 7200
+    },
+    {
+      "epoch": 0.6478077647269793,
+      "grad_norm": 1.702125683289223,
+      "learning_rate": 7.827039952176946e-06,
+      "loss": 0.7696,
+      "step": 7225
+    },
+    {
+      "epoch": 0.6500493140858962,
+      "grad_norm": 1.7917713066235164,
+      "learning_rate": 7.77722427020026e-06,
+      "loss": 0.7664,
+      "step": 7250
+    },
+    {
+      "epoch": 0.652290863444813,
+      "grad_norm": 1.6937229241150964,
+      "learning_rate": 7.727408588223574e-06,
+      "loss": 0.7515,
+      "step": 7275
+    },
+    {
+      "epoch": 0.65453241280373,
+      "grad_norm": 1.7775884772395607,
+      "learning_rate": 7.677592906246887e-06,
+      "loss": 0.767,
+      "step": 7300
+    },
+    {
+      "epoch": 0.6567739621626468,
+      "grad_norm": 1.7226500388022883,
+      "learning_rate": 7.6277772242702006e-06,
+      "loss": 0.7676,
+      "step": 7325
+    },
+    {
+      "epoch": 0.6590155115215637,
+      "grad_norm": 1.738366084601186,
+      "learning_rate": 7.577961542293515e-06,
+      "loss": 0.7796,
+      "step": 7350
+    },
+    {
+      "epoch": 0.6612570608804806,
+      "grad_norm": 1.659677524471436,
+      "learning_rate": 7.5281458603168275e-06,
+      "loss": 0.7543,
+      "step": 7375
+    },
+    {
+      "epoch": 0.6634986102393975,
+      "grad_norm": 1.7220186075774808,
+      "learning_rate": 7.478330178340142e-06,
+      "loss": 0.7918,
+      "step": 7400
+    },
+    {
+      "epoch": 0.6657401595983143,
+      "grad_norm": 1.7274393080929407,
+      "learning_rate": 7.428514496363456e-06,
+      "loss": 0.7882,
+      "step": 7425
+    },
+    {
+      "epoch": 0.6679817089572312,
+      "grad_norm": 1.8016655237128902,
+      "learning_rate": 7.37869881438677e-06,
+      "loss": 0.7889,
+      "step": 7450
+    },
+    {
+      "epoch": 0.6702232583161482,
+      "grad_norm": 1.9514224447470465,
+      "learning_rate": 7.328883132410083e-06,
+      "loss": 0.7772,
+      "step": 7475
+    },
+    {
+      "epoch": 0.672464807675065,
+      "grad_norm": 1.6866541471123346,
+      "learning_rate": 7.279067450433397e-06,
+      "loss": 0.7537,
+      "step": 7500
+    },
+    {
+      "epoch": 0.6747063570339819,
+      "grad_norm": 1.5995131098013584,
+      "learning_rate": 7.229251768456711e-06,
+      "loss": 0.741,
+      "step": 7525
+    },
+    {
+      "epoch": 0.6769479063928988,
+      "grad_norm": 1.7025523689523283,
+      "learning_rate": 7.179436086480025e-06,
+      "loss": 0.7558,
+      "step": 7550
+    },
+    {
+      "epoch": 0.6791894557518157,
+      "grad_norm": 2.0095499267364745,
+      "learning_rate": 7.129620404503338e-06,
+      "loss": 0.757,
+      "step": 7575
+    },
+    {
+      "epoch": 0.6814310051107325,
+      "grad_norm": 1.7992263340730072,
+      "learning_rate": 7.0798047225266516e-06,
+      "loss": 0.7636,
+      "step": 7600
+    },
+    {
+      "epoch": 0.6836725544696495,
+      "grad_norm": 1.7559061906827926,
+      "learning_rate": 7.029989040549966e-06,
+      "loss": 0.778,
+      "step": 7625
+    },
+    {
+      "epoch": 0.6859141038285663,
+      "grad_norm": 1.956755336390013,
+      "learning_rate": 6.980173358573279e-06,
+      "loss": 0.7683,
+      "step": 7650
+    },
+    {
+      "epoch": 0.6881556531874832,
+      "grad_norm": 1.7448110825187608,
+      "learning_rate": 6.930357676596593e-06,
+      "loss": 0.7751,
+      "step": 7675
+    },
+    {
+      "epoch": 0.6903972025464,
+      "grad_norm": 1.6670481706384996,
+      "learning_rate": 6.880541994619907e-06,
+      "loss": 0.7628,
+      "step": 7700
+    },
+    {
+      "epoch": 0.692638751905317,
+      "grad_norm": 1.7735206213020065,
+      "learning_rate": 6.830726312643221e-06,
+      "loss": 0.7667,
+      "step": 7725
+    },
+    {
+      "epoch": 0.6948803012642338,
+      "grad_norm": 1.6570338157726383,
+      "learning_rate": 6.780910630666534e-06,
+      "loss": 0.7373,
+      "step": 7750
+    },
+    {
+      "epoch": 0.6971218506231507,
+      "grad_norm": 1.8540028960496349,
+      "learning_rate": 6.731094948689848e-06,
+      "loss": 0.7319,
+      "step": 7775
+    },
+    {
+      "epoch": 0.6993633999820676,
+      "grad_norm": 1.6857468789586427,
+      "learning_rate": 6.681279266713162e-06,
+      "loss": 0.7632,
+      "step": 7800
+    },
+    {
+      "epoch": 0.7016049493409845,
+      "grad_norm": 1.6890850810078746,
+      "learning_rate": 6.6314635847364765e-06,
+      "loss": 0.7749,
+      "step": 7825
+    },
+    {
+      "epoch": 0.7038464986999013,
+      "grad_norm": 1.924422917379529,
+      "learning_rate": 6.581647902759789e-06,
+      "loss": 0.7583,
+      "step": 7850
+    },
+    {
+      "epoch": 0.7060880480588183,
+      "grad_norm": 1.630090078651694,
+      "learning_rate": 6.531832220783103e-06,
+      "loss": 0.7631,
+      "step": 7875
+    },
+    {
+      "epoch": 0.7083295974177352,
+      "grad_norm": 2.0468696598695804,
+      "learning_rate": 6.482016538806417e-06,
+      "loss": 0.7353,
+      "step": 7900
+    },
+    {
+      "epoch": 0.710571146776652,
+      "grad_norm": 1.8259752815108847,
+      "learning_rate": 6.43220085682973e-06,
+      "loss": 0.7549,
+      "step": 7925
+    },
+    {
+      "epoch": 0.7128126961355689,
+      "grad_norm": 1.9661073390983392,
+      "learning_rate": 6.382385174853044e-06,
+      "loss": 0.7387,
+      "step": 7950
+    },
+    {
+      "epoch": 0.7150542454944858,
+      "grad_norm": 1.8203344976450027,
+      "learning_rate": 6.332569492876358e-06,
+      "loss": 0.7503,
+      "step": 7975
+    },
+    {
+      "epoch": 0.7172957948534027,
+      "grad_norm": 1.6706387021687052,
+      "learning_rate": 6.282753810899672e-06,
+      "loss": 0.7547,
+      "step": 8000
+    },
+    {
+      "epoch": 0.7195373442123195,
+      "grad_norm": 1.6604842538698719,
+      "learning_rate": 6.232938128922985e-06,
+      "loss": 0.7572,
+      "step": 8025
+    },
+    {
+      "epoch": 0.7217788935712365,
+      "grad_norm": 1.6447415350716448,
+      "learning_rate": 6.183122446946299e-06,
+      "loss": 0.7296,
+      "step": 8050
+    },
+    {
+      "epoch": 0.7240204429301533,
+      "grad_norm": 1.9020553686330788,
+      "learning_rate": 6.133306764969613e-06,
+      "loss": 0.7487,
+      "step": 8075
+    },
+    {
+      "epoch": 0.7262619922890702,
+      "grad_norm": 1.8432810146288983,
+      "learning_rate": 6.083491082992926e-06,
+      "loss": 0.7536,
+      "step": 8100
+    },
+    {
+      "epoch": 0.7285035416479871,
+      "grad_norm": 1.6720677963590904,
+      "learning_rate": 6.0336754010162405e-06,
+      "loss": 0.7503,
+      "step": 8125
+    },
+    {
+      "epoch": 0.730745091006904,
+      "grad_norm": 1.6924073549958094,
+      "learning_rate": 5.983859719039554e-06,
+      "loss": 0.7311,
+      "step": 8150
+    },
+    {
+      "epoch": 0.7329866403658208,
+      "grad_norm": 1.7039808290333063,
+      "learning_rate": 5.934044037062868e-06,
+      "loss": 0.7328,
+      "step": 8175
+    },
+    {
+      "epoch": 0.7352281897247377,
+      "grad_norm": 1.8427531227740936,
+      "learning_rate": 5.884228355086181e-06,
+      "loss": 0.7425,
+      "step": 8200
+    },
+    {
+      "epoch": 0.7374697390836547,
+      "grad_norm": 1.7942833275137888,
+      "learning_rate": 5.834412673109495e-06,
+      "loss": 0.7267,
+      "step": 8225
+    },
+    {
+      "epoch": 0.7397112884425715,
+      "grad_norm": 1.7022537282468946,
+      "learning_rate": 5.784596991132809e-06,
+      "loss": 0.7272,
+      "step": 8250
+    },
+    {
+      "epoch": 0.7419528378014884,
+      "grad_norm": 1.7117727723387872,
+      "learning_rate": 5.734781309156123e-06,
+      "loss": 0.7406,
+      "step": 8275
+    },
+    {
+      "epoch": 0.7441943871604053,
+      "grad_norm": 1.6662930795240602,
+      "learning_rate": 5.684965627179436e-06,
+      "loss": 0.7144,
+      "step": 8300
+    },
+    {
+      "epoch": 0.7464359365193222,
+      "grad_norm": 1.7271646882114742,
+      "learning_rate": 5.63514994520275e-06,
+      "loss": 0.7194,
+      "step": 8325
+    },
+    {
+      "epoch": 0.748677485878239,
+      "grad_norm": 1.7848763942941896,
+      "learning_rate": 5.585334263226065e-06,
+      "loss": 0.746,
+      "step": 8350
+    },
+    {
+      "epoch": 0.750919035237156,
+      "grad_norm": 1.8023379110361575,
+      "learning_rate": 5.535518581249378e-06,
+      "loss": 0.7626,
+      "step": 8375
+    },
+    {
+      "epoch": 0.7531605845960728,
+      "grad_norm": 1.912633778788871,
+      "learning_rate": 5.4857028992726915e-06,
+      "loss": 0.742,
+      "step": 8400
+    },
+    {
+      "epoch": 0.7554021339549897,
+      "grad_norm": 1.7489417073434457,
+      "learning_rate": 5.4358872172960054e-06,
+      "loss": 0.7235,
+      "step": 8425
+    },
+    {
+      "epoch": 0.7576436833139065,
+      "grad_norm": 1.8578170309164246,
+      "learning_rate": 5.386071535319319e-06,
+      "loss": 0.7406,
+      "step": 8450
+    },
+    {
+      "epoch": 0.7598852326728235,
+      "grad_norm": 1.6841301357346117,
+      "learning_rate": 5.336255853342632e-06,
+      "loss": 0.7265,
+      "step": 8475
+    },
+    {
+      "epoch": 0.7621267820317403,
+      "grad_norm": 1.67637322338657,
+      "learning_rate": 5.286440171365946e-06,
+      "loss": 0.7298,
+      "step": 8500
+    },
+    {
+      "epoch": 0.7643683313906572,
+      "grad_norm": 2.75057542787619,
+      "learning_rate": 5.23662448938926e-06,
+      "loss": 0.7403,
+      "step": 8525
+    },
+    {
+      "epoch": 0.7666098807495741,
+      "grad_norm": 1.9562182621837296,
+      "learning_rate": 5.186808807412575e-06,
+      "loss": 0.7357,
+      "step": 8550
+    },
+    {
+      "epoch": 0.768851430108491,
+      "grad_norm": 1.6178018797694105,
+      "learning_rate": 5.136993125435887e-06,
+      "loss": 0.7204,
+      "step": 8575
+    },
+    {
+      "epoch": 0.7710929794674078,
+      "grad_norm": 1.7743524698802162,
+      "learning_rate": 5.087177443459202e-06,
+      "loss": 0.7184,
+      "step": 8600
+    },
+    {
+      "epoch": 0.7733345288263248,
+      "grad_norm": 1.7898971028849646,
+      "learning_rate": 5.037361761482516e-06,
+      "loss": 0.7175,
+      "step": 8625
+    },
+    {
+      "epoch": 0.7755760781852417,
+      "grad_norm": 1.6656106823591226,
+      "learning_rate": 4.987546079505829e-06,
+      "loss": 0.7321,
+      "step": 8650
+    },
+    {
+      "epoch": 0.7778176275441585,
+      "grad_norm": 1.7374976494979955,
+      "learning_rate": 4.9377303975291425e-06,
+      "loss": 0.7143,
+      "step": 8675
+    },
+    {
+      "epoch": 0.7800591769030754,
+      "grad_norm": 1.7005970489827344,
+      "learning_rate": 4.8879147155524564e-06,
+      "loss": 0.7192,
+      "step": 8700
+    },
+    {
+      "epoch": 0.7823007262619923,
+      "grad_norm": 1.9487288387163249,
+      "learning_rate": 4.83809903357577e-06,
+      "loss": 0.7316,
+      "step": 8725
+    },
+    {
+      "epoch": 0.7845422756209092,
+      "grad_norm": 1.925164538441118,
+      "learning_rate": 4.788283351599084e-06,
+      "loss": 0.7168,
+      "step": 8750
+    },
+    {
+      "epoch": 0.786783824979826,
+      "grad_norm": 1.8020535072536237,
+      "learning_rate": 4.738467669622397e-06,
+      "loss": 0.717,
+      "step": 8775
+    },
+    {
+      "epoch": 0.789025374338743,
+      "grad_norm": 1.7020258141919171,
+      "learning_rate": 4.688651987645711e-06,
+      "loss": 0.705,
+      "step": 8800
+    },
+    {
+      "epoch": 0.7912669236976598,
+      "grad_norm": 1.7601750027580583,
+      "learning_rate": 4.638836305669025e-06,
+      "loss": 0.7182,
+      "step": 8825
+    },
+    {
+      "epoch": 0.7935084730565767,
+      "grad_norm": 1.846265294881566,
+      "learning_rate": 4.589020623692339e-06,
+      "loss": 0.7201,
+      "step": 8850
+    },
+    {
+      "epoch": 0.7957500224154936,
+      "grad_norm": 1.923703748637079,
+      "learning_rate": 4.539204941715653e-06,
+      "loss": 0.7197,
+      "step": 8875
+    },
+    {
+      "epoch": 0.7979915717744105,
+      "grad_norm": 1.8056553901529664,
+      "learning_rate": 4.489389259738966e-06,
+      "loss": 0.7285,
+      "step": 8900
+    },
+    {
+      "epoch": 0.8002331211333273,
+      "grad_norm": 1.795629570041314,
+      "learning_rate": 4.43957357776228e-06,
+      "loss": 0.7163,
+      "step": 8925
+    },
+    {
+      "epoch": 0.8024746704922442,
+      "grad_norm": 1.5666589265984843,
+      "learning_rate": 4.3897578957855936e-06,
+      "loss": 0.7275,
+      "step": 8950
+    },
+    {
+      "epoch": 0.8047162198511612,
+      "grad_norm": 1.6549818049918694,
+      "learning_rate": 4.3399422138089074e-06,
+      "loss": 0.7039,
+      "step": 8975
+    },
+    {
+      "epoch": 0.806957769210078,
+      "grad_norm": 1.7814051375647009,
+      "learning_rate": 4.290126531832221e-06,
+      "loss": 0.7227,
+      "step": 9000
+    },
+    {
+      "epoch": 0.8091993185689949,
+      "grad_norm": 1.7898138695626653,
+      "learning_rate": 4.240310849855535e-06,
+      "loss": 0.7079,
+      "step": 9025
+    },
+    {
+      "epoch": 0.8114408679279118,
+      "grad_norm": 1.515676572777515,
+      "learning_rate": 4.190495167878848e-06,
+      "loss": 0.7045,
+      "step": 9050
+    },
+    {
+      "epoch": 0.8136824172868287,
+      "grad_norm": 1.898283603072741,
+      "learning_rate": 4.140679485902162e-06,
+      "loss": 0.7,
+      "step": 9075
+    },
+    {
+      "epoch": 0.8159239666457455,
+      "grad_norm": 1.7281337925852416,
+      "learning_rate": 4.090863803925476e-06,
+      "loss": 0.7286,
+      "step": 9100
+    },
+    {
+      "epoch": 0.8181655160046625,
+      "grad_norm": 1.8301510420483158,
+      "learning_rate": 4.04104812194879e-06,
+      "loss": 0.7238,
+      "step": 9125
+    },
+    {
+      "epoch": 0.8204070653635793,
+      "grad_norm": 1.784168091420267,
+      "learning_rate": 3.991232439972104e-06,
+      "loss": 0.7141,
+      "step": 9150
+    },
+    {
+      "epoch": 0.8226486147224962,
+      "grad_norm": 1.7711618071268116,
+      "learning_rate": 3.941416757995417e-06,
+      "loss": 0.705,
+      "step": 9175
+    },
+    {
+      "epoch": 0.824890164081413,
+      "grad_norm": 1.7943262394506765,
+      "learning_rate": 3.8916010760187315e-06,
+      "loss": 0.6912,
+      "step": 9200
+    },
+    {
+      "epoch": 0.82713171344033,
+      "grad_norm": 1.7222088978012928,
+      "learning_rate": 3.8417853940420446e-06,
+      "loss": 0.7113,
+      "step": 9225
+    },
+    {
+      "epoch": 0.8293732627992468,
+      "grad_norm": 1.7435568457969255,
+      "learning_rate": 3.791969712065359e-06,
+      "loss": 0.7057,
+      "step": 9250
+    },
+    {
+      "epoch": 0.8316148121581637,
+      "grad_norm": 1.6373991331602138,
+      "learning_rate": 3.7421540300886723e-06,
+      "loss": 0.7125,
+      "step": 9275
+    },
+    {
+      "epoch": 0.8338563615170806,
+      "grad_norm": 1.9154506347265303,
+      "learning_rate": 3.692338348111986e-06,
+      "loss": 0.7061,
+      "step": 9300
+    },
+    {
+      "epoch": 0.8360979108759975,
+      "grad_norm": 1.6555518537057525,
+      "learning_rate": 3.6425226661352997e-06,
+      "loss": 0.7067,
+      "step": 9325
+    },
+    {
+      "epoch": 0.8383394602349143,
+      "grad_norm": 1.6884565793905875,
+      "learning_rate": 3.592706984158613e-06,
+      "loss": 0.6977,
+      "step": 9350
+    },
+    {
+      "epoch": 0.8405810095938313,
+      "grad_norm": 1.840174884218591,
+      "learning_rate": 3.5428913021819274e-06,
+      "loss": 0.705,
+      "step": 9375
+    },
+    {
+      "epoch": 0.8428225589527482,
+      "grad_norm": 1.7593245437402398,
+      "learning_rate": 3.493075620205241e-06,
+      "loss": 0.7009,
+      "step": 9400
+    },
+    {
+      "epoch": 0.845064108311665,
+      "grad_norm": 1.758774652432913,
+      "learning_rate": 3.4432599382285548e-06,
+      "loss": 0.7098,
+      "step": 9425
+    },
+    {
+      "epoch": 0.8473056576705819,
+      "grad_norm": 2.005892733446428,
+      "learning_rate": 3.3934442562518682e-06,
+      "loss": 0.6976,
+      "step": 9450
+    },
+    {
+      "epoch": 0.8495472070294988,
+      "grad_norm": 1.9488435196357063,
+      "learning_rate": 3.343628574275182e-06,
+      "loss": 0.6847,
+      "step": 9475
+    },
+    {
+      "epoch": 0.8517887563884157,
+      "grad_norm": 1.6521048790538413,
+      "learning_rate": 3.2938128922984956e-06,
+      "loss": 0.7175,
+      "step": 9500
+    },
+    {
+      "epoch": 0.8540303057473325,
+      "grad_norm": 1.7212515172137823,
+      "learning_rate": 3.24399721032181e-06,
+      "loss": 0.6974,
+      "step": 9525
+    },
+    {
+      "epoch": 0.8562718551062495,
+      "grad_norm": 1.7737509134642555,
+      "learning_rate": 3.1941815283451233e-06,
+      "loss": 0.708,
+      "step": 9550
+    },
+    {
+      "epoch": 0.8585134044651663,
+      "grad_norm": 1.7653120750571727,
+      "learning_rate": 3.1443658463684368e-06,
+      "loss": 0.7159,
+      "step": 9575
+    },
+    {
+      "epoch": 0.8607549538240832,
+      "grad_norm": 1.734111952523256,
+      "learning_rate": 3.0945501643917507e-06,
+      "loss": 0.7077,
+      "step": 9600
+    },
+    {
+      "epoch": 0.8629965031830001,
+      "grad_norm": 1.7456697538522787,
+      "learning_rate": 3.044734482415064e-06,
+      "loss": 0.7087,
+      "step": 9625
+    },
+    {
+      "epoch": 0.865238052541917,
+      "grad_norm": 1.7238213295739973,
+      "learning_rate": 2.9949188004383784e-06,
+      "loss": 0.7005,
+      "step": 9650
+    },
+    {
+      "epoch": 0.8674796019008338,
+      "grad_norm": 1.8377474317274547,
+      "learning_rate": 2.945103118461692e-06,
+      "loss": 0.6944,
+      "step": 9675
+    },
+    {
+      "epoch": 0.8697211512597507,
+      "grad_norm": 1.7142002887130772,
+      "learning_rate": 2.8952874364850058e-06,
+      "loss": 0.7053,
+      "step": 9700
+    },
+    {
+      "epoch": 0.8719627006186677,
+      "grad_norm": 1.7300662647642029,
+      "learning_rate": 2.8454717545083192e-06,
+      "loss": 0.6977,
+      "step": 9725
+    },
+    {
+      "epoch": 0.8742042499775845,
+      "grad_norm": 1.7629299601559292,
+      "learning_rate": 2.7956560725316335e-06,
+      "loss": 0.7258,
+      "step": 9750
+    },
+    {
+      "epoch": 0.8764457993365014,
+      "grad_norm": 2.035565282704572,
+      "learning_rate": 2.745840390554947e-06,
+      "loss": 0.6958,
+      "step": 9775
+    },
+    {
+      "epoch": 0.8786873486954183,
+      "grad_norm": 1.9591996552629682,
+      "learning_rate": 2.696024708578261e-06,
+      "loss": 0.7007,
+      "step": 9800
+    },
+    {
+      "epoch": 0.8809288980543352,
+      "grad_norm": 1.7485828313310643,
+      "learning_rate": 2.6462090266015743e-06,
+      "loss": 0.7155,
+      "step": 9825
+    },
+    {
+      "epoch": 0.883170447413252,
+      "grad_norm": 1.7931319821271166,
+      "learning_rate": 2.596393344624888e-06,
+      "loss": 0.6986,
+      "step": 9850
+    },
+    {
+      "epoch": 0.885411996772169,
+      "grad_norm": 1.9307468792470688,
+      "learning_rate": 2.546577662648202e-06,
+      "loss": 0.6803,
+      "step": 9875
+    },
+    {
+      "epoch": 0.8876535461310858,
+      "grad_norm": 1.7491594363238476,
+      "learning_rate": 2.4967619806715156e-06,
+      "loss": 0.6906,
+      "step": 9900
+    },
+    {
+      "epoch": 0.8898950954900027,
+      "grad_norm": 1.6753791987181879,
+      "learning_rate": 2.4469462986948294e-06,
+      "loss": 0.6852,
+      "step": 9925
+    },
+    {
+      "epoch": 0.8921366448489195,
+      "grad_norm": 2.0182721863038804,
+      "learning_rate": 2.397130616718143e-06,
+      "loss": 0.6905,
+      "step": 9950
+    },
+    {
+      "epoch": 0.8943781942078365,
+      "grad_norm": 1.8187532671275537,
+      "learning_rate": 2.3473149347414568e-06,
+      "loss": 0.699,
+      "step": 9975
+    },
+    {
+      "epoch": 0.8966197435667533,
+      "grad_norm": 1.6663248832177007,
+      "learning_rate": 2.2974992527647707e-06,
+      "loss": 0.7003,
+      "step": 10000
+    },
+    {
+      "epoch": 0.8988612929256702,
+      "grad_norm": 1.9202844832714863,
+      "learning_rate": 2.247683570788084e-06,
+      "loss": 0.6835,
+      "step": 10025
+    },
+    {
+      "epoch": 0.9011028422845871,
+      "grad_norm": 1.6853858508241173,
+      "learning_rate": 2.197867888811398e-06,
+      "loss": 0.6916,
+      "step": 10050
+    },
+    {
+      "epoch": 0.903344391643504,
+      "grad_norm": 1.9868266435863697,
+      "learning_rate": 2.148052206834712e-06,
+      "loss": 0.6844,
+      "step": 10075
+    },
+    {
+      "epoch": 0.9055859410024208,
+      "grad_norm": 1.8849355749170535,
+      "learning_rate": 2.0982365248580253e-06,
+      "loss": 0.7072,
+      "step": 10100
+    },
+    {
+      "epoch": 0.9078274903613378,
+      "grad_norm": 1.6731153407780723,
+      "learning_rate": 2.0484208428813392e-06,
+      "loss": 0.6607,
+      "step": 10125
+    },
+    {
+      "epoch": 0.9100690397202547,
+      "grad_norm": 1.5128757018080325,
+      "learning_rate": 1.998605160904653e-06,
+      "loss": 0.6918,
+      "step": 10150
+    },
+    {
+      "epoch": 0.9123105890791715,
+      "grad_norm": 1.767646778725392,
+      "learning_rate": 1.9487894789279666e-06,
+      "loss": 0.6961,
+      "step": 10175
+    },
+    {
+      "epoch": 0.9145521384380884,
+      "grad_norm": 1.6315602541613066,
+      "learning_rate": 1.8989737969512802e-06,
+      "loss": 0.6693,
+      "step": 10200
+    },
+    {
+      "epoch": 0.9167936877970053,
+      "grad_norm": 1.8678688697190617,
+      "learning_rate": 1.8491581149745941e-06,
+      "loss": 0.6922,
+      "step": 10225
+    },
+    {
+      "epoch": 0.9190352371559222,
+      "grad_norm": 1.611712478036576,
+      "learning_rate": 1.7993424329979078e-06,
+      "loss": 0.6811,
+      "step": 10250
+    },
+    {
+      "epoch": 0.921276786514839,
+      "grad_norm": 1.6843339133008968,
+      "learning_rate": 1.7495267510212217e-06,
+      "loss": 0.6755,
+      "step": 10275
+    },
+    {
+      "epoch": 0.923518335873756,
+      "grad_norm": 1.6194366447525863,
+      "learning_rate": 1.6997110690445353e-06,
+      "loss": 0.6812,
+      "step": 10300
+    },
+    {
+      "epoch": 0.9257598852326728,
+      "grad_norm": 1.860646662108025,
+      "learning_rate": 1.6498953870678492e-06,
+      "loss": 0.6969,
+      "step": 10325
+    },
+    {
+      "epoch": 0.9280014345915897,
+      "grad_norm": 2.0713345944656907,
+      "learning_rate": 1.6000797050911629e-06,
+      "loss": 0.6728,
+      "step": 10350
+    },
+    {
+      "epoch": 0.9302429839505066,
+      "grad_norm": 1.8840185563281908,
+      "learning_rate": 1.5502640231144768e-06,
+      "loss": 0.6745,
+      "step": 10375
+    },
+    {
+      "epoch": 0.9324845333094235,
+      "grad_norm": 1.938783236028355,
+      "learning_rate": 1.5004483411377904e-06,
+      "loss": 0.6662,
+      "step": 10400
+    },
+    {
+      "epoch": 0.9347260826683403,
+      "grad_norm": 1.5778651962299233,
+      "learning_rate": 1.4506326591611039e-06,
+      "loss": 0.6677,
+      "step": 10425
+    },
+    {
+      "epoch": 0.9369676320272572,
+      "grad_norm": 1.841716703463937,
+      "learning_rate": 1.4008169771844178e-06,
+      "loss": 0.7011,
+      "step": 10450
+    },
+    {
+      "epoch": 0.9392091813861742,
+      "grad_norm": 1.7110561420246093,
+      "learning_rate": 1.3510012952077314e-06,
+      "loss": 0.6917,
+      "step": 10475
+    },
+    {
+      "epoch": 0.941450730745091,
+      "grad_norm": 1.9073072846225319,
+      "learning_rate": 1.3011856132310451e-06,
+      "loss": 0.6819,
+      "step": 10500
+    },
+    {
+      "epoch": 0.9436922801040079,
+      "grad_norm": 1.9505417382169639,
+      "learning_rate": 1.251369931254359e-06,
+      "loss": 0.6775,
+      "step": 10525
+    },
+    {
+      "epoch": 0.9459338294629248,
+      "grad_norm": 1.7446591705196661,
+      "learning_rate": 1.2015542492776727e-06,
+      "loss": 0.6689,
+      "step": 10550
+    },
+    {
+      "epoch": 0.9481753788218417,
+      "grad_norm": 1.750359764202537,
+      "learning_rate": 1.1517385673009865e-06,
+      "loss": 0.6961,
+      "step": 10575
+    },
+    {
+      "epoch": 0.9504169281807585,
+      "grad_norm": 1.7291009933484693,
+      "learning_rate": 1.1019228853243002e-06,
+      "loss": 0.6826,
+      "step": 10600
+    },
+    {
+      "epoch": 0.9526584775396755,
+      "grad_norm": 1.632647267687145,
+      "learning_rate": 1.0521072033476139e-06,
+      "loss": 0.664,
+      "step": 10625
+    },
+    {
+      "epoch": 0.9549000268985923,
+      "grad_norm": 1.7161704518625818,
+      "learning_rate": 1.0022915213709276e-06,
+      "loss": 0.6859,
+      "step": 10650
+    },
+    {
+      "epoch": 0.9571415762575092,
+      "grad_norm": 1.6401263166918565,
+      "learning_rate": 9.524758393942414e-07,
+      "loss": 0.6786,
+      "step": 10675
+    },
+    {
+      "epoch": 0.959383125616426,
+      "grad_norm": 1.7482513109597007,
+      "learning_rate": 9.026601574175552e-07,
+      "loss": 0.6663,
+      "step": 10700
+    },
+    {
+      "epoch": 0.961624674975343,
+      "grad_norm": 1.684959994399361,
+      "learning_rate": 8.528444754408688e-07,
+      "loss": 0.6752,
+      "step": 10725
+    },
+    {
+      "epoch": 0.9638662243342598,
+      "grad_norm": 1.8573881688476663,
+      "learning_rate": 8.030287934641826e-07,
+      "loss": 0.6843,
+      "step": 10750
+    },
+    {
+      "epoch": 0.9661077736931767,
+      "grad_norm": 1.8062245924885205,
+      "learning_rate": 7.532131114874963e-07,
+      "loss": 0.6863,
+      "step": 10775
+    },
+    {
+      "epoch": 0.9683493230520936,
+      "grad_norm": 1.7107706105839677,
+      "learning_rate": 7.033974295108101e-07,
+      "loss": 0.6934,
+      "step": 10800
+    },
+    {
+      "epoch": 0.9705908724110105,
+      "grad_norm": 1.7092942609036264,
+      "learning_rate": 6.535817475341239e-07,
+      "loss": 0.6577,
+      "step": 10825
+    },
+    {
+      "epoch": 0.9728324217699273,
+      "grad_norm": 1.7804059319379388,
+      "learning_rate": 6.037660655574375e-07,
+      "loss": 0.6699,
+      "step": 10850
+    },
+    {
+      "epoch": 0.9750739711288443,
+      "grad_norm": 1.8514662312474457,
+      "learning_rate": 5.539503835807512e-07,
+      "loss": 0.6628,
+      "step": 10875
+    },
+    {
+      "epoch": 0.9773155204877612,
+      "grad_norm": 1.7868336576505317,
+      "learning_rate": 5.04134701604065e-07,
+      "loss": 0.6979,
+      "step": 10900
+    },
+    {
+      "epoch": 0.979557069846678,
+      "grad_norm": 1.7157675627877909,
+      "learning_rate": 4.5431901962737877e-07,
+      "loss": 0.681,
+      "step": 10925
+    },
+    {
+      "epoch": 0.9817986192055949,
+      "grad_norm": 1.7269621963297628,
+      "learning_rate": 4.0450333765069244e-07,
+      "loss": 0.6748,
+      "step": 10950
+    },
+    {
+      "epoch": 0.9840401685645118,
+      "grad_norm": 1.6849545670128994,
+      "learning_rate": 3.546876556740062e-07,
+      "loss": 0.6845,
+      "step": 10975
+    },
+    {
+      "epoch": 0.9862817179234287,
+      "grad_norm": 1.712207675718165,
+      "learning_rate": 3.0487197369731994e-07,
+      "loss": 0.6648,
+      "step": 11000
+    },
+    {
+      "epoch": 0.9885232672823455,
+      "grad_norm": 1.8999442305937304,
+      "learning_rate": 2.5505629172063366e-07,
+      "loss": 0.6558,
+      "step": 11025
+    },
+    {
+      "epoch": 0.9907648166412625,
+      "grad_norm": 1.699393044053189,
+      "learning_rate": 2.052406097439474e-07,
+      "loss": 0.6864,
+      "step": 11050
+    },
+    {
+      "epoch": 0.9930063660001793,
+      "grad_norm": 1.7110416398737864,
+      "learning_rate": 1.5542492776726116e-07,
+      "loss": 0.6724,
+      "step": 11075
+    },
+    {
+      "epoch": 0.9952479153590962,
+      "grad_norm": 1.7936965958729694,
+      "learning_rate": 1.0560924579057488e-07,
+      "loss": 0.6524,
+      "step": 11100
+    },
+    {
+      "epoch": 0.9974894647180131,
+      "grad_norm": 1.793636959019703,
+      "learning_rate": 5.5793563813888614e-08,
+      "loss": 0.6882,
+      "step": 11125
+    },
+    {
+      "epoch": 0.99973101407693,
+      "grad_norm": 1.658624597079087,
+      "learning_rate": 5.9778818372023515e-09,
+      "loss": 0.6742,
+      "step": 11150
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 11153,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 583802824949760.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-11153/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b1d0e0bfafc928d0ee97b04e6b80105bb4334369cd7dcde0e604159bf51c593
+size 6072

checkpoint-11153/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "meta-llama/Meta-Llama-3-8B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.0",
+  "use_cache": true,
+  "vocab_size": 128264
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "transformers_version": "4.40.0"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4dc60ba1d2073bdd95a0f469265587e9711e0e35b5569da784f287cd6ab0356
+size 4976764208

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cb22b1acddca151372ab19dda1bc7dfed4848c70e15bf93e20fc7c02b4fdb98
+size 4999802720

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5136aa7410834a6af82d77dbbb7b3e854b09b750e55406a7c6b3ced0e24b4afb
+size 4915916176

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8708006d0d25d48da829f87f16dd42ba10f7286b63ab292b7546c45be3ff2059
+size 1168204344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16060653568
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,13 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>user",
+    "<|im_start|>assistant",
+    "<|im_start|>system",
+    "<|im_end|>",
+    "<s>",
+    "<pad>"
+  ],
+  "bos_token": "<s>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<pad>"
+}

tokenizer_config.json

@@ -0,0 +1,2119 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128256": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128257": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128258": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128259": {
+      "content": "<|im_start|>user",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128260": {
+      "content": "<|im_start|>assistant",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128261": {
+      "content": "<|im_start|>system",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>user",
+    "<|im_start|>assistant",
+    "<|im_start|>system",
+    "<|im_end|>",
+    "<s>",
+    "<pad>"
+  ],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% if loop.last and add_generation_prompt %}{{'<|im_start|>assistant\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|im_end|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 8192,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b1d0e0bfafc928d0ee97b04e6b80105bb4334369cd7dcde0e604159bf51c593
+size 6072

training_params.json

@@ -0,0 +1,47 @@
+{
+    "model": "meta-llama/Meta-Llama-3-8B-Instruct",
+    "project_name": "autotrain-llama3-oh1",
+    "data_path": "HuggingFaceH4/OpenHermes-2.5",
+    "train_split": "train_sft",
+    "valid_split": null,
+    "add_eos_token": false,
+    "block_size": 1024,
+    "model_max_length": 8192,
+    "padding": "right",
+    "trainer": "sft",
+    "use_flash_attention_2": false,
+    "log": "none",
+    "disable_gradient_checkpointing": false,
+    "logging_steps": -1,
+    "evaluation_strategy": "epoch",
+    "save_total_limit": 1,
+    "save_strategy": "epoch",
+    "auto_find_batch_size": false,
+    "mixed_precision": "bf16",
+    "lr": 2e-05,
+    "epochs": 1,
+    "batch_size": 4,
+    "warmup_ratio": 0.1,
+    "gradient_accumulation": 1,
+    "optimizer": "adamw_torch",
+    "scheduler": "linear",
+    "weight_decay": 0.0,
+    "max_grad_norm": 1.0,
+    "seed": 42,
+    "chat_template": "chatml",
+    "quantization": null,
+    "target_modules": null,
+    "merge_adapter": false,
+    "peft": false,
+    "lora_r": 16,
+    "lora_alpha": 32,
+    "lora_dropout": 0.05,
+    "model_ref": null,
+    "dpo_beta": 0.1,
+    "prompt_text_column": "prompt",
+    "text_column": "messages",
+    "rejected_text_column": "rejected",
+    "push_to_hub": true,
+    "repo_id": "abhishek/autotrain-llama3-oh1",
+    "username": "abhishek"
+}