Upload folder using huggingface_hub

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 irish_llama31_8b_data_v3/checkpoint-1900/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/tokenizer.json filter=lfs diff=lfs merge=lfs -text

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.1",
+  "use_cache": true,
+  "vocab_size": 128256
+}

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.95,
+  "transformers_version": "4.46.1"
+}

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/latest

@@ -0,0 +1 @@
+global_step1002

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/pytorch_model-00001-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d6a4afb2d557d8f5ad31dae2ebcb86e5f59c33d48f8b428c65ff24297e15f1a
+size 4976717314

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/pytorch_model-00002-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:005ca0a4b81eca3239cd50ff7abaf9e35291df38a779e6f9a766175703b48c9f
+size 4999826246

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/pytorch_model-00003-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dda700360fe8706855fe421ad8928a2ccb09b4cdac659dc645b01f92170b221b
+size 4915938762

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/pytorch_model-00004-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09cb520783a65e5e017ceab7876e501405b8095d8a2a654ed931391ab1da3ea1
+size 1168140873

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/pytorch_model.bin.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16060522496
+  },
+  "weight_map": {
+    "lm_head.weight": "pytorch_model-00004-of-00004.bin",
+    "model.embed_tokens.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.10.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.19.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.19.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.19.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.20.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.20.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.21.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.28.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.28.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.28.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.28.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.29.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.29.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.29.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.29.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.30.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.30.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.30.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.30.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.31.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.31.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.31.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.31.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.9.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.norm.weight": "pytorch_model-00004-of-00004.bin"
+  }
+}

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5aaaf172fac11e3d2eecaf6ba53ac6a40ff6781bf91fe38b5a2bf5f4cf4742e
+size 14512

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef3b803afeee75f294af23b48130432f78c5777137654dd7d5b89c4f68fea58a
+size 14512

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e353e7da0fe193ec052db80148f8bea749a0c31eea0cb6d26b7045b913368ae5
+size 1064

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<｜end▁of▁sentence｜>"
+}

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d91915040cfac999d8c55f4b5bc6e67367c065e3a7a4e4b9438ce1f256addd86
+size 17209530

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/tokenizer_config.json

@@ -0,0 +1,2066 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<｜begin▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<｜end▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<｜User｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128012": {
+      "content": "<｜Assistant｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128013": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128014": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "128015": {
+      "content": "<｜▁pad▁｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\\n'}}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<｜end▁of▁sentence｜>",
+  "legacy": true,
+  "model_max_length": 16384,
+  "pad_token": "<｜end▁of▁sentence｜>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": null,
+  "use_default_system_prompt": false
+}

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/trainer_state.json

@@ -0,0 +1,3547 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9966355140186915,
+  "eval_steps": 500,
+  "global_step": 1002,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0029906542056074765,
+      "grad_norm": 3.4282846450805664,
+      "learning_rate": 1.9607843137254904e-07,
+      "loss": 0.8056,
+      "step": 1
+    },
+    {
+      "epoch": 0.005981308411214953,
+      "grad_norm": 4.388183116912842,
+      "learning_rate": 3.921568627450981e-07,
+      "loss": 0.8628,
+      "step": 2
+    },
+    {
+      "epoch": 0.011962616822429906,
+      "grad_norm": 5.366924285888672,
+      "learning_rate": 7.843137254901962e-07,
+      "loss": 0.8594,
+      "step": 4
+    },
+    {
+      "epoch": 0.01794392523364486,
+      "grad_norm": 4.848186492919922,
+      "learning_rate": 1.1764705882352942e-06,
+      "loss": 0.8382,
+      "step": 6
+    },
+    {
+      "epoch": 0.023925233644859812,
+      "grad_norm": 4.195390701293945,
+      "learning_rate": 1.5686274509803923e-06,
+      "loss": 0.776,
+      "step": 8
+    },
+    {
+      "epoch": 0.029906542056074768,
+      "grad_norm": 4.072334289550781,
+      "learning_rate": 1.96078431372549e-06,
+      "loss": 0.8074,
+      "step": 10
+    },
+    {
+      "epoch": 0.03588785046728972,
+      "grad_norm": 2.706496477127075,
+      "learning_rate": 2.3529411764705885e-06,
+      "loss": 0.7487,
+      "step": 12
+    },
+    {
+      "epoch": 0.041869158878504675,
+      "grad_norm": 1.9595896005630493,
+      "learning_rate": 2.7450980392156867e-06,
+      "loss": 0.7245,
+      "step": 14
+    },
+    {
+      "epoch": 0.047850467289719624,
+      "grad_norm": 2.018717050552368,
+      "learning_rate": 3.1372549019607846e-06,
+      "loss": 0.7219,
+      "step": 16
+    },
+    {
+      "epoch": 0.05383177570093458,
+      "grad_norm": 1.7622108459472656,
+      "learning_rate": 3.529411764705883e-06,
+      "loss": 0.6933,
+      "step": 18
+    },
+    {
+      "epoch": 0.059813084112149535,
+      "grad_norm": 1.6138136386871338,
+      "learning_rate": 3.92156862745098e-06,
+      "loss": 0.6903,
+      "step": 20
+    },
+    {
+      "epoch": 0.06579439252336448,
+      "grad_norm": 1.403164029121399,
+      "learning_rate": 4.313725490196079e-06,
+      "loss": 0.6621,
+      "step": 22
+    },
+    {
+      "epoch": 0.07177570093457944,
+      "grad_norm": 1.144256353378296,
+      "learning_rate": 4.705882352941177e-06,
+      "loss": 0.6622,
+      "step": 24
+    },
+    {
+      "epoch": 0.0777570093457944,
+      "grad_norm": 1.123942255973816,
+      "learning_rate": 5.098039215686274e-06,
+      "loss": 0.6382,
+      "step": 26
+    },
+    {
+      "epoch": 0.08373831775700935,
+      "grad_norm": 0.8241124153137207,
+      "learning_rate": 5.4901960784313735e-06,
+      "loss": 0.6438,
+      "step": 28
+    },
+    {
+      "epoch": 0.08971962616822429,
+      "grad_norm": 0.7706263661384583,
+      "learning_rate": 5.882352941176471e-06,
+      "loss": 0.5828,
+      "step": 30
+    },
+    {
+      "epoch": 0.09570093457943925,
+      "grad_norm": 0.7647000551223755,
+      "learning_rate": 6.274509803921569e-06,
+      "loss": 0.5619,
+      "step": 32
+    },
+    {
+      "epoch": 0.1016822429906542,
+      "grad_norm": 3.782670021057129,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.6539,
+      "step": 34
+    },
+    {
+      "epoch": 0.10766355140186916,
+      "grad_norm": 0.6794511675834656,
+      "learning_rate": 7.058823529411766e-06,
+      "loss": 0.5877,
+      "step": 36
+    },
+    {
+      "epoch": 0.11364485981308411,
+      "grad_norm": 0.6795441508293152,
+      "learning_rate": 7.450980392156863e-06,
+      "loss": 0.5926,
+      "step": 38
+    },
+    {
+      "epoch": 0.11962616822429907,
+      "grad_norm": 0.6446499228477478,
+      "learning_rate": 7.84313725490196e-06,
+      "loss": 0.5877,
+      "step": 40
+    },
+    {
+      "epoch": 0.12560747663551403,
+      "grad_norm": 0.6556646227836609,
+      "learning_rate": 8.23529411764706e-06,
+      "loss": 0.5971,
+      "step": 42
+    },
+    {
+      "epoch": 0.13158878504672897,
+      "grad_norm": 0.6358281970024109,
+      "learning_rate": 8.627450980392157e-06,
+      "loss": 0.5903,
+      "step": 44
+    },
+    {
+      "epoch": 0.13757009345794394,
+      "grad_norm": 0.5848496556282043,
+      "learning_rate": 9.019607843137256e-06,
+      "loss": 0.5673,
+      "step": 46
+    },
+    {
+      "epoch": 0.14355140186915888,
+      "grad_norm": 0.6347646117210388,
+      "learning_rate": 9.411764705882354e-06,
+      "loss": 0.596,
+      "step": 48
+    },
+    {
+      "epoch": 0.14953271028037382,
+      "grad_norm": 0.6139919757843018,
+      "learning_rate": 9.803921568627451e-06,
+      "loss": 0.5652,
+      "step": 50
+    },
+    {
+      "epoch": 0.1555140186915888,
+      "grad_norm": 0.6299135088920593,
+      "learning_rate": 9.999972717866783e-06,
+      "loss": 0.6019,
+      "step": 52
+    },
+    {
+      "epoch": 0.16149532710280373,
+      "grad_norm": 0.6271386742591858,
+      "learning_rate": 9.999754462587396e-06,
+      "loss": 0.5812,
+      "step": 54
+    },
+    {
+      "epoch": 0.1674766355140187,
+      "grad_norm": 0.603743851184845,
+      "learning_rate": 9.999317961555746e-06,
+      "loss": 0.5837,
+      "step": 56
+    },
+    {
+      "epoch": 0.17345794392523364,
+      "grad_norm": 0.6218274235725403,
+      "learning_rate": 9.998663233825668e-06,
+      "loss": 0.5741,
+      "step": 58
+    },
+    {
+      "epoch": 0.17943925233644858,
+      "grad_norm": 0.5972462296485901,
+      "learning_rate": 9.997790307976874e-06,
+      "loss": 0.53,
+      "step": 60
+    },
+    {
+      "epoch": 0.18542056074766355,
+      "grad_norm": 0.5708372592926025,
+      "learning_rate": 9.996699222113709e-06,
+      "loss": 0.5526,
+      "step": 62
+    },
+    {
+      "epoch": 0.1914018691588785,
+      "grad_norm": 0.5606462955474854,
+      "learning_rate": 9.995390023863483e-06,
+      "loss": 0.5534,
+      "step": 64
+    },
+    {
+      "epoch": 0.19738317757009347,
+      "grad_norm": 0.6092550158500671,
+      "learning_rate": 9.99386277037439e-06,
+      "loss": 0.5766,
+      "step": 66
+    },
+    {
+      "epoch": 0.2033644859813084,
+      "grad_norm": 0.577298104763031,
+      "learning_rate": 9.992117528313025e-06,
+      "loss": 0.5611,
+      "step": 68
+    },
+    {
+      "epoch": 0.20934579439252338,
+      "grad_norm": 0.5903265476226807,
+      "learning_rate": 9.990154373861463e-06,
+      "loss": 0.5929,
+      "step": 70
+    },
+    {
+      "epoch": 0.21532710280373832,
+      "grad_norm": 0.5991240739822388,
+      "learning_rate": 9.987973392713932e-06,
+      "loss": 0.5676,
+      "step": 72
+    },
+    {
+      "epoch": 0.22130841121495326,
+      "grad_norm": 0.6097000241279602,
+      "learning_rate": 9.985574680073084e-06,
+      "loss": 0.5349,
+      "step": 74
+    },
+    {
+      "epoch": 0.22728971962616823,
+      "grad_norm": 0.5570875406265259,
+      "learning_rate": 9.982958340645832e-06,
+      "loss": 0.5671,
+      "step": 76
+    },
+    {
+      "epoch": 0.23327102803738317,
+      "grad_norm": 0.5734003186225891,
+      "learning_rate": 9.980124488638774e-06,
+      "loss": 0.5955,
+      "step": 78
+    },
+    {
+      "epoch": 0.23925233644859814,
+      "grad_norm": 0.5406553745269775,
+      "learning_rate": 9.977073247753218e-06,
+      "loss": 0.5366,
+      "step": 80
+    },
+    {
+      "epoch": 0.24523364485981308,
+      "grad_norm": 1.5509494543075562,
+      "learning_rate": 9.973804751179779e-06,
+      "loss": 0.5512,
+      "step": 82
+    },
+    {
+      "epoch": 0.25121495327102805,
+      "grad_norm": 0.5826510787010193,
+      "learning_rate": 9.970319141592559e-06,
+      "loss": 0.5441,
+      "step": 84
+    },
+    {
+      "epoch": 0.25719626168224297,
+      "grad_norm": 0.5949900150299072,
+      "learning_rate": 9.966616571142931e-06,
+      "loss": 0.5626,
+      "step": 86
+    },
+    {
+      "epoch": 0.26317757009345794,
+      "grad_norm": 0.5483706593513489,
+      "learning_rate": 9.962697201452882e-06,
+      "loss": 0.5465,
+      "step": 88
+    },
+    {
+      "epoch": 0.2691588785046729,
+      "grad_norm": 0.6367583274841309,
+      "learning_rate": 9.958561203607975e-06,
+      "loss": 0.5404,
+      "step": 90
+    },
+    {
+      "epoch": 0.2751401869158879,
+      "grad_norm": 0.539474606513977,
+      "learning_rate": 9.954208758149867e-06,
+      "loss": 0.5477,
+      "step": 92
+    },
+    {
+      "epoch": 0.2811214953271028,
+      "grad_norm": 0.5971343517303467,
+      "learning_rate": 9.949640055068432e-06,
+      "loss": 0.5568,
+      "step": 94
+    },
+    {
+      "epoch": 0.28710280373831776,
+      "grad_norm": 0.6063545346260071,
+      "learning_rate": 9.944855293793477e-06,
+      "loss": 0.5576,
+      "step": 96
+    },
+    {
+      "epoch": 0.2930841121495327,
+      "grad_norm": 0.572651207447052,
+      "learning_rate": 9.939854683186016e-06,
+      "loss": 0.5583,
+      "step": 98
+    },
+    {
+      "epoch": 0.29906542056074764,
+      "grad_norm": 0.5835568308830261,
+      "learning_rate": 9.93463844152918e-06,
+      "loss": 0.5843,
+      "step": 100
+    },
+    {
+      "epoch": 0.3050467289719626,
+      "grad_norm": 0.5889995694160461,
+      "learning_rate": 9.929206796518663e-06,
+      "loss": 0.5314,
+      "step": 102
+    },
+    {
+      "epoch": 0.3110280373831776,
+      "grad_norm": 0.5598423480987549,
+      "learning_rate": 9.9235599852528e-06,
+      "loss": 0.5315,
+      "step": 104
+    },
+    {
+      "epoch": 0.31700934579439255,
+      "grad_norm": 0.565233051776886,
+      "learning_rate": 9.91769825422221e-06,
+      "loss": 0.5428,
+      "step": 106
+    },
+    {
+      "epoch": 0.32299065420560746,
+      "grad_norm": 0.5561599135398865,
+      "learning_rate": 9.91162185929904e-06,
+      "loss": 0.5273,
+      "step": 108
+    },
+    {
+      "epoch": 0.32897196261682243,
+      "grad_norm": 0.5615388751029968,
+      "learning_rate": 9.905331065725789e-06,
+      "loss": 0.5645,
+      "step": 110
+    },
+    {
+      "epoch": 0.3349532710280374,
+      "grad_norm": 0.5586528182029724,
+      "learning_rate": 9.898826148103737e-06,
+      "loss": 0.5343,
+      "step": 112
+    },
+    {
+      "epoch": 0.3409345794392523,
+      "grad_norm": 0.5534612536430359,
+      "learning_rate": 9.892107390380959e-06,
+      "loss": 0.5231,
+      "step": 114
+    },
+    {
+      "epoch": 0.3469158878504673,
+      "grad_norm": 0.6707253456115723,
+      "learning_rate": 9.88517508583992e-06,
+      "loss": 0.5546,
+      "step": 116
+    },
+    {
+      "epoch": 0.35289719626168226,
+      "grad_norm": 0.5770431160926819,
+      "learning_rate": 9.878029537084685e-06,
+      "loss": 0.5539,
+      "step": 118
+    },
+    {
+      "epoch": 0.35887850467289717,
+      "grad_norm": 0.5553021430969238,
+      "learning_rate": 9.870671056027705e-06,
+      "loss": 0.544,
+      "step": 120
+    },
+    {
+      "epoch": 0.36485981308411214,
+      "grad_norm": 0.5602825880050659,
+      "learning_rate": 9.863099963876202e-06,
+      "loss": 0.5458,
+      "step": 122
+    },
+    {
+      "epoch": 0.3708411214953271,
+      "grad_norm": 0.5332139134407043,
+      "learning_rate": 9.855316591118143e-06,
+      "loss": 0.5605,
+      "step": 124
+    },
+    {
+      "epoch": 0.3768224299065421,
+      "grad_norm": 0.603239893913269,
+      "learning_rate": 9.847321277507821e-06,
+      "loss": 0.5378,
+      "step": 126
+    },
+    {
+      "epoch": 0.382803738317757,
+      "grad_norm": 0.5638384222984314,
+      "learning_rate": 9.839114372051025e-06,
+      "loss": 0.5451,
+      "step": 128
+    },
+    {
+      "epoch": 0.38878504672897196,
+      "grad_norm": 0.5452869534492493,
+      "learning_rate": 9.830696232989793e-06,
+      "loss": 0.5248,
+      "step": 130
+    },
+    {
+      "epoch": 0.39476635514018693,
+      "grad_norm": 2.0323398113250732,
+      "learning_rate": 9.822067227786794e-06,
+      "loss": 0.5345,
+      "step": 132
+    },
+    {
+      "epoch": 0.40074766355140184,
+      "grad_norm": 0.5626135468482971,
+      "learning_rate": 9.813227733109267e-06,
+      "loss": 0.5462,
+      "step": 134
+    },
+    {
+      "epoch": 0.4067289719626168,
+      "grad_norm": 0.6065902709960938,
+      "learning_rate": 9.804178134812597e-06,
+      "loss": 0.5553,
+      "step": 136
+    },
+    {
+      "epoch": 0.4127102803738318,
+      "grad_norm": 0.5674175024032593,
+      "learning_rate": 9.794918827923458e-06,
+      "loss": 0.555,
+      "step": 138
+    },
+    {
+      "epoch": 0.41869158878504675,
+      "grad_norm": 0.5876524448394775,
+      "learning_rate": 9.785450216622579e-06,
+      "loss": 0.5449,
+      "step": 140
+    },
+    {
+      "epoch": 0.42467289719626167,
+      "grad_norm": 0.5519018173217773,
+      "learning_rate": 9.775772714227096e-06,
+      "loss": 0.5486,
+      "step": 142
+    },
+    {
+      "epoch": 0.43065420560747664,
+      "grad_norm": 0.5821079611778259,
+      "learning_rate": 9.765886743172512e-06,
+      "loss": 0.555,
+      "step": 144
+    },
+    {
+      "epoch": 0.4366355140186916,
+      "grad_norm": 0.5479001402854919,
+      "learning_rate": 9.755792734994257e-06,
+      "loss": 0.5521,
+      "step": 146
+    },
+    {
+      "epoch": 0.4426168224299065,
+      "grad_norm": 0.5903741717338562,
+      "learning_rate": 9.745491130308848e-06,
+      "loss": 0.5247,
+      "step": 148
+    },
+    {
+      "epoch": 0.4485981308411215,
+      "grad_norm": 0.5696671605110168,
+      "learning_rate": 9.734982378794662e-06,
+      "loss": 0.5067,
+      "step": 150
+    },
+    {
+      "epoch": 0.45457943925233646,
+      "grad_norm": 0.5558380484580994,
+      "learning_rate": 9.724266939172302e-06,
+      "loss": 0.5103,
+      "step": 152
+    },
+    {
+      "epoch": 0.46056074766355143,
+      "grad_norm": 0.5476086735725403,
+      "learning_rate": 9.713345279184574e-06,
+      "loss": 0.5465,
+      "step": 154
+    },
+    {
+      "epoch": 0.46654205607476634,
+      "grad_norm": 0.536656379699707,
+      "learning_rate": 9.702217875576067e-06,
+      "loss": 0.5531,
+      "step": 156
+    },
+    {
+      "epoch": 0.4725233644859813,
+      "grad_norm": 0.5399542450904846,
+      "learning_rate": 9.690885214072352e-06,
+      "loss": 0.5258,
+      "step": 158
+    },
+    {
+      "epoch": 0.4785046728971963,
+      "grad_norm": 0.539088785648346,
+      "learning_rate": 9.679347789358766e-06,
+      "loss": 0.5311,
+      "step": 160
+    },
+    {
+      "epoch": 0.4844859813084112,
+      "grad_norm": 0.5658231973648071,
+      "learning_rate": 9.667606105058828e-06,
+      "loss": 0.5522,
+      "step": 162
+    },
+    {
+      "epoch": 0.49046728971962616,
+      "grad_norm": 0.5501983761787415,
+      "learning_rate": 9.655660673712254e-06,
+      "loss": 0.4861,
+      "step": 164
+    },
+    {
+      "epoch": 0.49644859813084113,
+      "grad_norm": 0.5300902128219604,
+      "learning_rate": 9.643512016752578e-06,
+      "loss": 0.5325,
+      "step": 166
+    },
+    {
+      "epoch": 0.5024299065420561,
+      "grad_norm": 0.5249874591827393,
+      "learning_rate": 9.631160664484398e-06,
+      "loss": 0.5308,
+      "step": 168
+    },
+    {
+      "epoch": 0.508411214953271,
+      "grad_norm": 0.502623438835144,
+      "learning_rate": 9.618607156060227e-06,
+      "loss": 0.5296,
+      "step": 170
+    },
+    {
+      "epoch": 0.5143925233644859,
+      "grad_norm": 0.5451452136039734,
+      "learning_rate": 9.605852039456947e-06,
+      "loss": 0.5447,
+      "step": 172
+    },
+    {
+      "epoch": 0.520373831775701,
+      "grad_norm": 0.5635679364204407,
+      "learning_rate": 9.592895871451908e-06,
+      "loss": 0.5505,
+      "step": 174
+    },
+    {
+      "epoch": 0.5263551401869159,
+      "grad_norm": 0.8386549353599548,
+      "learning_rate": 9.579739217598608e-06,
+      "loss": 0.5495,
+      "step": 176
+    },
+    {
+      "epoch": 0.5323364485981309,
+      "grad_norm": 0.5260588526725769,
+      "learning_rate": 9.566382652202013e-06,
+      "loss": 0.5223,
+      "step": 178
+    },
+    {
+      "epoch": 0.5383177570093458,
+      "grad_norm": 0.5311717987060547,
+      "learning_rate": 9.552826758293487e-06,
+      "loss": 0.5479,
+      "step": 180
+    },
+    {
+      "epoch": 0.5442990654205607,
+      "grad_norm": 0.5487819910049438,
+      "learning_rate": 9.539072127605342e-06,
+      "loss": 0.52,
+      "step": 182
+    },
+    {
+      "epoch": 0.5502803738317757,
+      "grad_norm": 0.5302957892417908,
+      "learning_rate": 9.525119360545005e-06,
+      "loss": 0.539,
+      "step": 184
+    },
+    {
+      "epoch": 0.5562616822429907,
+      "grad_norm": 0.5473540425300598,
+      "learning_rate": 9.510969066168814e-06,
+      "loss": 0.5432,
+      "step": 186
+    },
+    {
+      "epoch": 0.5622429906542056,
+      "grad_norm": 0.5388506650924683,
+      "learning_rate": 9.496621862155434e-06,
+      "loss": 0.5472,
+      "step": 188
+    },
+    {
+      "epoch": 0.5682242990654206,
+      "grad_norm": 0.533203125,
+      "learning_rate": 9.482078374778877e-06,
+      "loss": 0.5365,
+      "step": 190
+    },
+    {
+      "epoch": 0.5742056074766355,
+      "grad_norm": 0.5521941781044006,
+      "learning_rate": 9.467339238881199e-06,
+      "loss": 0.548,
+      "step": 192
+    },
+    {
+      "epoch": 0.5801869158878504,
+      "grad_norm": 0.5113306045532227,
+      "learning_rate": 9.452405097844746e-06,
+      "loss": 0.5294,
+      "step": 194
+    },
+    {
+      "epoch": 0.5861682242990655,
+      "grad_norm": 0.5288083553314209,
+      "learning_rate": 9.437276603564104e-06,
+      "loss": 0.5344,
+      "step": 196
+    },
+    {
+      "epoch": 0.5921495327102804,
+      "grad_norm": 0.5461395978927612,
+      "learning_rate": 9.421954416417624e-06,
+      "loss": 0.5446,
+      "step": 198
+    },
+    {
+      "epoch": 0.5981308411214953,
+      "grad_norm": 0.5455833077430725,
+      "learning_rate": 9.406439205238604e-06,
+      "loss": 0.5165,
+      "step": 200
+    },
+    {
+      "epoch": 0.6041121495327103,
+      "grad_norm": 0.5564857125282288,
+      "learning_rate": 9.39073164728609e-06,
+      "loss": 0.5374,
+      "step": 202
+    },
+    {
+      "epoch": 0.6100934579439252,
+      "grad_norm": 0.5761339068412781,
+      "learning_rate": 9.37483242821531e-06,
+      "loss": 0.5408,
+      "step": 204
+    },
+    {
+      "epoch": 0.6160747663551401,
+      "grad_norm": 0.5414023995399475,
+      "learning_rate": 9.358742242047754e-06,
+      "loss": 0.5302,
+      "step": 206
+    },
+    {
+      "epoch": 0.6220560747663552,
+      "grad_norm": 0.580768346786499,
+      "learning_rate": 9.342461791140869e-06,
+      "loss": 0.5307,
+      "step": 208
+    },
+    {
+      "epoch": 0.6280373831775701,
+      "grad_norm": 0.543178915977478,
+      "learning_rate": 9.325991786157405e-06,
+      "loss": 0.5372,
+      "step": 210
+    },
+    {
+      "epoch": 0.6340186915887851,
+      "grad_norm": 0.5230343341827393,
+      "learning_rate": 9.309332946034392e-06,
+      "loss": 0.5124,
+      "step": 212
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 0.5364699959754944,
+      "learning_rate": 9.292485997951759e-06,
+      "loss": 0.5214,
+      "step": 214
+    },
+    {
+      "epoch": 0.6459813084112149,
+      "grad_norm": 0.5475817918777466,
+      "learning_rate": 9.275451677300591e-06,
+      "loss": 0.558,
+      "step": 216
+    },
+    {
+      "epoch": 0.65196261682243,
+      "grad_norm": 0.5522270798683167,
+      "learning_rate": 9.258230727651029e-06,
+      "loss": 0.5202,
+      "step": 218
+    },
+    {
+      "epoch": 0.6579439252336449,
+      "grad_norm": 0.5273387432098389,
+      "learning_rate": 9.240823900719807e-06,
+      "loss": 0.538,
+      "step": 220
+    },
+    {
+      "epoch": 0.6639252336448598,
+      "grad_norm": 0.541516125202179,
+      "learning_rate": 9.22323195633745e-06,
+      "loss": 0.5166,
+      "step": 222
+    },
+    {
+      "epoch": 0.6699065420560748,
+      "grad_norm": 0.5298076272010803,
+      "learning_rate": 9.205455662415093e-06,
+      "loss": 0.5294,
+      "step": 224
+    },
+    {
+      "epoch": 0.6758878504672897,
+      "grad_norm": 0.5472598075866699,
+      "learning_rate": 9.187495794910967e-06,
+      "loss": 0.5329,
+      "step": 226
+    },
+    {
+      "epoch": 0.6818691588785046,
+      "grad_norm": 0.5340124368667603,
+      "learning_rate": 9.169353137796533e-06,
+      "loss": 0.5312,
+      "step": 228
+    },
+    {
+      "epoch": 0.6878504672897197,
+      "grad_norm": 0.5367527604103088,
+      "learning_rate": 9.151028483022253e-06,
+      "loss": 0.5471,
+      "step": 230
+    },
+    {
+      "epoch": 0.6938317757009346,
+      "grad_norm": 0.5586390495300293,
+      "learning_rate": 9.132522630483018e-06,
+      "loss": 0.5364,
+      "step": 232
+    },
+    {
+      "epoch": 0.6998130841121495,
+      "grad_norm": 0.5255227088928223,
+      "learning_rate": 9.113836387983239e-06,
+      "loss": 0.5225,
+      "step": 234
+    },
+    {
+      "epoch": 0.7057943925233645,
+      "grad_norm": 0.5368794798851013,
+      "learning_rate": 9.094970571201582e-06,
+      "loss": 0.5523,
+      "step": 236
+    },
+    {
+      "epoch": 0.7117757009345794,
+      "grad_norm": 0.5149601101875305,
+      "learning_rate": 9.075926003655366e-06,
+      "loss": 0.5084,
+      "step": 238
+    },
+    {
+      "epoch": 0.7177570093457943,
+      "grad_norm": 0.5425019860267639,
+      "learning_rate": 9.056703516664606e-06,
+      "loss": 0.5352,
+      "step": 240
+    },
+    {
+      "epoch": 0.7237383177570094,
+      "grad_norm": 0.5437155365943909,
+      "learning_rate": 9.037303949315737e-06,
+      "loss": 0.5343,
+      "step": 242
+    },
+    {
+      "epoch": 0.7297196261682243,
+      "grad_norm": 0.523941159248352,
+      "learning_rate": 9.017728148424975e-06,
+      "loss": 0.5501,
+      "step": 244
+    },
+    {
+      "epoch": 0.7357009345794393,
+      "grad_norm": 0.526647686958313,
+      "learning_rate": 8.997976968501362e-06,
+      "loss": 0.5484,
+      "step": 246
+    },
+    {
+      "epoch": 0.7416822429906542,
+      "grad_norm": 0.5470868945121765,
+      "learning_rate": 8.97805127170946e-06,
+      "loss": 0.5401,
+      "step": 248
+    },
+    {
+      "epoch": 0.7476635514018691,
+      "grad_norm": 0.523993194103241,
+      "learning_rate": 8.957951927831722e-06,
+      "loss": 0.5411,
+      "step": 250
+    },
+    {
+      "epoch": 0.7536448598130842,
+      "grad_norm": 0.5142015218734741,
+      "learning_rate": 8.937679814230517e-06,
+      "loss": 0.531,
+      "step": 252
+    },
+    {
+      "epoch": 0.7596261682242991,
+      "grad_norm": 0.5653587579727173,
+      "learning_rate": 8.917235815809835e-06,
+      "loss": 0.53,
+      "step": 254
+    },
+    {
+      "epoch": 0.765607476635514,
+      "grad_norm": 0.5519165396690369,
+      "learning_rate": 8.896620824976665e-06,
+      "loss": 0.5311,
+      "step": 256
+    },
+    {
+      "epoch": 0.771588785046729,
+      "grad_norm": 0.5087413191795349,
+      "learning_rate": 8.875835741602031e-06,
+      "loss": 0.4977,
+      "step": 258
+    },
+    {
+      "epoch": 0.7775700934579439,
+      "grad_norm": 0.5064166188240051,
+      "learning_rate": 8.854881472981722e-06,
+      "loss": 0.5174,
+      "step": 260
+    },
+    {
+      "epoch": 0.7835514018691588,
+      "grad_norm": 0.5295135378837585,
+      "learning_rate": 8.833758933796679e-06,
+      "loss": 0.5233,
+      "step": 262
+    },
+    {
+      "epoch": 0.7895327102803739,
+      "grad_norm": 0.5436890721321106,
+      "learning_rate": 8.812469046073069e-06,
+      "loss": 0.5213,
+      "step": 264
+    },
+    {
+      "epoch": 0.7955140186915888,
+      "grad_norm": 0.528203547000885,
+      "learning_rate": 8.791012739142045e-06,
+      "loss": 0.537,
+      "step": 266
+    },
+    {
+      "epoch": 0.8014953271028037,
+      "grad_norm": 0.5551559329032898,
+      "learning_rate": 8.769390949599166e-06,
+      "loss": 0.5559,
+      "step": 268
+    },
+    {
+      "epoch": 0.8074766355140187,
+      "grad_norm": 0.5346433520317078,
+      "learning_rate": 8.74760462126353e-06,
+      "loss": 0.5533,
+      "step": 270
+    },
+    {
+      "epoch": 0.8134579439252336,
+      "grad_norm": 0.5381712317466736,
+      "learning_rate": 8.725654705136557e-06,
+      "loss": 0.5364,
+      "step": 272
+    },
+    {
+      "epoch": 0.8194392523364487,
+      "grad_norm": 0.5317867994308472,
+      "learning_rate": 8.703542159360495e-06,
+      "loss": 0.5408,
+      "step": 274
+    },
+    {
+      "epoch": 0.8254205607476636,
+      "grad_norm": 0.51235032081604,
+      "learning_rate": 8.68126794917658e-06,
+      "loss": 0.5078,
+      "step": 276
+    },
+    {
+      "epoch": 0.8314018691588785,
+      "grad_norm": 0.5308533310890198,
+      "learning_rate": 8.658833046882909e-06,
+      "loss": 0.5441,
+      "step": 278
+    },
+    {
+      "epoch": 0.8373831775700935,
+      "grad_norm": 0.5278391242027283,
+      "learning_rate": 8.636238431792002e-06,
+      "loss": 0.5065,
+      "step": 280
+    },
+    {
+      "epoch": 0.8433644859813084,
+      "grad_norm": 0.5376297831535339,
+      "learning_rate": 8.613485090188044e-06,
+      "loss": 0.5509,
+      "step": 282
+    },
+    {
+      "epoch": 0.8493457943925233,
+      "grad_norm": 0.5556045174598694,
+      "learning_rate": 8.590574015283841e-06,
+      "loss": 0.5506,
+      "step": 284
+    },
+    {
+      "epoch": 0.8553271028037384,
+      "grad_norm": 0.5278720259666443,
+      "learning_rate": 8.567506207177461e-06,
+      "loss": 0.5296,
+      "step": 286
+    },
+    {
+      "epoch": 0.8613084112149533,
+      "grad_norm": 0.5656425356864929,
+      "learning_rate": 8.54428267280858e-06,
+      "loss": 0.5491,
+      "step": 288
+    },
+    {
+      "epoch": 0.8672897196261682,
+      "grad_norm": 0.5447366833686829,
+      "learning_rate": 8.520904425914525e-06,
+      "loss": 0.5368,
+      "step": 290
+    },
+    {
+      "epoch": 0.8732710280373832,
+      "grad_norm": 0.5339160561561584,
+      "learning_rate": 8.497372486986024e-06,
+      "loss": 0.5419,
+      "step": 292
+    },
+    {
+      "epoch": 0.8792523364485981,
+      "grad_norm": 0.5290377736091614,
+      "learning_rate": 8.473687883222665e-06,
+      "loss": 0.5413,
+      "step": 294
+    },
+    {
+      "epoch": 0.885233644859813,
+      "grad_norm": 0.5392821431159973,
+      "learning_rate": 8.449851648488051e-06,
+      "loss": 0.5479,
+      "step": 296
+    },
+    {
+      "epoch": 0.8912149532710281,
+      "grad_norm": 0.552673876285553,
+      "learning_rate": 8.425864823264674e-06,
+      "loss": 0.5407,
+      "step": 298
+    },
+    {
+      "epoch": 0.897196261682243,
+      "grad_norm": 1.8682053089141846,
+      "learning_rate": 8.401728454608495e-06,
+      "loss": 0.5366,
+      "step": 300
+    },
+    {
+      "epoch": 0.9031775700934579,
+      "grad_norm": 0.51381915807724,
+      "learning_rate": 8.377443596103238e-06,
+      "loss": 0.5201,
+      "step": 302
+    },
+    {
+      "epoch": 0.9091588785046729,
+      "grad_norm": 0.5145431756973267,
+      "learning_rate": 8.353011307814405e-06,
+      "loss": 0.5174,
+      "step": 304
+    },
+    {
+      "epoch": 0.9151401869158878,
+      "grad_norm": 0.5476198196411133,
+      "learning_rate": 8.328432656242998e-06,
+      "loss": 0.5323,
+      "step": 306
+    },
+    {
+      "epoch": 0.9211214953271029,
+      "grad_norm": 0.5376638770103455,
+      "learning_rate": 8.303708714278959e-06,
+      "loss": 0.5216,
+      "step": 308
+    },
+    {
+      "epoch": 0.9271028037383178,
+      "grad_norm": 0.5454867482185364,
+      "learning_rate": 8.278840561154354e-06,
+      "loss": 0.5257,
+      "step": 310
+    },
+    {
+      "epoch": 0.9330841121495327,
+      "grad_norm": 0.5648353099822998,
+      "learning_rate": 8.253829282396246e-06,
+      "loss": 0.5196,
+      "step": 312
+    },
+    {
+      "epoch": 0.9390654205607477,
+      "grad_norm": 0.5360947847366333,
+      "learning_rate": 8.228675969779319e-06,
+      "loss": 0.5282,
+      "step": 314
+    },
+    {
+      "epoch": 0.9450467289719626,
+      "grad_norm": 0.49109888076782227,
+      "learning_rate": 8.203381721278216e-06,
+      "loss": 0.4976,
+      "step": 316
+    },
+    {
+      "epoch": 0.9510280373831775,
+      "grad_norm": 0.5353028774261475,
+      "learning_rate": 8.177947641019622e-06,
+      "loss": 0.53,
+      "step": 318
+    },
+    {
+      "epoch": 0.9570093457943926,
+      "grad_norm": 0.5335257053375244,
+      "learning_rate": 8.152374839234048e-06,
+      "loss": 0.5335,
+      "step": 320
+    },
+    {
+      "epoch": 0.9629906542056075,
+      "grad_norm": 0.5343037247657776,
+      "learning_rate": 8.126664432207385e-06,
+      "loss": 0.519,
+      "step": 322
+    },
+    {
+      "epoch": 0.9689719626168224,
+      "grad_norm": 0.538314700126648,
+      "learning_rate": 8.100817542232175e-06,
+      "loss": 0.5389,
+      "step": 324
+    },
+    {
+      "epoch": 0.9749532710280374,
+      "grad_norm": 0.5235503315925598,
+      "learning_rate": 8.074835297558607e-06,
+      "loss": 0.5398,
+      "step": 326
+    },
+    {
+      "epoch": 0.9809345794392523,
+      "grad_norm": 0.5203934907913208,
+      "learning_rate": 8.048718832345288e-06,
+      "loss": 0.544,
+      "step": 328
+    },
+    {
+      "epoch": 0.9869158878504672,
+      "grad_norm": 0.4957960247993469,
+      "learning_rate": 8.02246928660972e-06,
+      "loss": 0.5113,
+      "step": 330
+    },
+    {
+      "epoch": 0.9928971962616823,
+      "grad_norm": 0.5159174799919128,
+      "learning_rate": 7.996087806178541e-06,
+      "loss": 0.5354,
+      "step": 332
+    },
+    {
+      "epoch": 0.9988785046728972,
+      "grad_norm": 0.531970739364624,
+      "learning_rate": 7.969575542637513e-06,
+      "loss": 0.5336,
+      "step": 334
+    },
+    {
+      "epoch": 1.0048598130841122,
+      "grad_norm": 0.6489409804344177,
+      "learning_rate": 7.942933653281245e-06,
+      "loss": 0.6029,
+      "step": 336
+    },
+    {
+      "epoch": 1.010841121495327,
+      "grad_norm": 0.5936563014984131,
+      "learning_rate": 7.916163301062687e-06,
+      "loss": 0.4627,
+      "step": 338
+    },
+    {
+      "epoch": 1.016822429906542,
+      "grad_norm": 0.720933735370636,
+      "learning_rate": 7.889265654542354e-06,
+      "loss": 0.4601,
+      "step": 340
+    },
+    {
+      "epoch": 1.022803738317757,
+      "grad_norm": 0.5771041512489319,
+      "learning_rate": 7.862241887837322e-06,
+      "loss": 0.4264,
+      "step": 342
+    },
+    {
+      "epoch": 1.0287850467289719,
+      "grad_norm": 0.5498257875442505,
+      "learning_rate": 7.835093180569982e-06,
+      "loss": 0.4324,
+      "step": 344
+    },
+    {
+      "epoch": 1.0347663551401869,
+      "grad_norm": 0.5726507902145386,
+      "learning_rate": 7.807820717816534e-06,
+      "loss": 0.4422,
+      "step": 346
+    },
+    {
+      "epoch": 1.040747663551402,
+      "grad_norm": 0.5894380211830139,
+      "learning_rate": 7.780425690055275e-06,
+      "loss": 0.4435,
+      "step": 348
+    },
+    {
+      "epoch": 1.0467289719626167,
+      "grad_norm": 0.5721791982650757,
+      "learning_rate": 7.75290929311461e-06,
+      "loss": 0.4225,
+      "step": 350
+    },
+    {
+      "epoch": 1.0527102803738317,
+      "grad_norm": 0.5583845376968384,
+      "learning_rate": 7.72527272812088e-06,
+      "loss": 0.4553,
+      "step": 352
+    },
+    {
+      "epoch": 1.0586915887850468,
+      "grad_norm": 0.5590948462486267,
+      "learning_rate": 7.697517201445906e-06,
+      "loss": 0.4255,
+      "step": 354
+    },
+    {
+      "epoch": 1.0646728971962616,
+      "grad_norm": 0.5416548252105713,
+      "learning_rate": 7.669643924654346e-06,
+      "loss": 0.4208,
+      "step": 356
+    },
+    {
+      "epoch": 1.0706542056074766,
+      "grad_norm": 0.5137345790863037,
+      "learning_rate": 7.641654114450803e-06,
+      "loss": 0.4171,
+      "step": 358
+    },
+    {
+      "epoch": 1.0766355140186916,
+      "grad_norm": 0.5574047565460205,
+      "learning_rate": 7.613548992626711e-06,
+      "loss": 0.4309,
+      "step": 360
+    },
+    {
+      "epoch": 1.0826168224299066,
+      "grad_norm": 0.5360638499259949,
+      "learning_rate": 7.585329786007006e-06,
+      "loss": 0.4381,
+      "step": 362
+    },
+    {
+      "epoch": 1.0885981308411214,
+      "grad_norm": 0.5528514385223389,
+      "learning_rate": 7.556997726396578e-06,
+      "loss": 0.4362,
+      "step": 364
+    },
+    {
+      "epoch": 1.0945794392523365,
+      "grad_norm": 0.5627298951148987,
+      "learning_rate": 7.528554050526489e-06,
+      "loss": 0.4256,
+      "step": 366
+    },
+    {
+      "epoch": 1.1005607476635515,
+      "grad_norm": 0.564423680305481,
+      "learning_rate": 7.500000000000001e-06,
+      "loss": 0.4253,
+      "step": 368
+    },
+    {
+      "epoch": 1.1065420560747663,
+      "grad_norm": 0.5412989258766174,
+      "learning_rate": 7.471336821238367e-06,
+      "loss": 0.4505,
+      "step": 370
+    },
+    {
+      "epoch": 1.1125233644859813,
+      "grad_norm": 0.5909019708633423,
+      "learning_rate": 7.442565765426436e-06,
+      "loss": 0.4565,
+      "step": 372
+    },
+    {
+      "epoch": 1.1185046728971964,
+      "grad_norm": 0.5458247065544128,
+      "learning_rate": 7.413688088458022e-06,
+      "loss": 0.4337,
+      "step": 374
+    },
+    {
+      "epoch": 1.1244859813084112,
+      "grad_norm": 0.5678550601005554,
+      "learning_rate": 7.384705050881096e-06,
+      "loss": 0.4447,
+      "step": 376
+    },
+    {
+      "epoch": 1.1304672897196262,
+      "grad_norm": 0.5694032907485962,
+      "learning_rate": 7.355617917842751e-06,
+      "loss": 0.4202,
+      "step": 378
+    },
+    {
+      "epoch": 1.1364485981308412,
+      "grad_norm": 0.5525908470153809,
+      "learning_rate": 7.326427959033988e-06,
+      "loss": 0.4224,
+      "step": 380
+    },
+    {
+      "epoch": 1.142429906542056,
+      "grad_norm": 0.5501400828361511,
+      "learning_rate": 7.297136448634282e-06,
+      "loss": 0.4386,
+      "step": 382
+    },
+    {
+      "epoch": 1.148411214953271,
+      "grad_norm": 0.549866259098053,
+      "learning_rate": 7.267744665255966e-06,
+      "loss": 0.4368,
+      "step": 384
+    },
+    {
+      "epoch": 1.154392523364486,
+      "grad_norm": 0.532667875289917,
+      "learning_rate": 7.2382538918884185e-06,
+      "loss": 0.4459,
+      "step": 386
+    },
+    {
+      "epoch": 1.1603738317757009,
+      "grad_norm": 0.5948196649551392,
+      "learning_rate": 7.20866541584206e-06,
+      "loss": 0.4495,
+      "step": 388
+    },
+    {
+      "epoch": 1.1663551401869159,
+      "grad_norm": 0.5659380555152893,
+      "learning_rate": 7.178980528692161e-06,
+      "loss": 0.4333,
+      "step": 390
+    },
+    {
+      "epoch": 1.172336448598131,
+      "grad_norm": 0.5794370770454407,
+      "learning_rate": 7.1492005262224585e-06,
+      "loss": 0.4541,
+      "step": 392
+    },
+    {
+      "epoch": 1.1783177570093457,
+      "grad_norm": 0.5396531820297241,
+      "learning_rate": 7.1193267083685945e-06,
+      "loss": 0.4235,
+      "step": 394
+    },
+    {
+      "epoch": 1.1842990654205607,
+      "grad_norm": 0.5651098489761353,
+      "learning_rate": 7.089360379161381e-06,
+      "loss": 0.437,
+      "step": 396
+    },
+    {
+      "epoch": 1.1902803738317758,
+      "grad_norm": 0.5672603845596313,
+      "learning_rate": 7.059302846669864e-06,
+      "loss": 0.4267,
+      "step": 398
+    },
+    {
+      "epoch": 1.1962616822429906,
+      "grad_norm": 0.6180233955383301,
+      "learning_rate": 7.029155422944234e-06,
+      "loss": 0.4269,
+      "step": 400
+    },
+    {
+      "epoch": 1.2022429906542056,
+      "grad_norm": 0.5725499987602234,
+      "learning_rate": 6.998919423958548e-06,
+      "loss": 0.4548,
+      "step": 402
+    },
+    {
+      "epoch": 1.2082242990654206,
+      "grad_norm": 0.5386544466018677,
+      "learning_rate": 6.9685961695532875e-06,
+      "loss": 0.42,
+      "step": 404
+    },
+    {
+      "epoch": 1.2142056074766354,
+      "grad_norm": 0.5460840463638306,
+      "learning_rate": 6.93818698337775e-06,
+      "loss": 0.4324,
+      "step": 406
+    },
+    {
+      "epoch": 1.2201869158878504,
+      "grad_norm": 0.5525109171867371,
+      "learning_rate": 6.907693192832263e-06,
+      "loss": 0.4419,
+      "step": 408
+    },
+    {
+      "epoch": 1.2261682242990655,
+      "grad_norm": 0.5663102269172668,
+      "learning_rate": 6.877116129010244e-06,
+      "loss": 0.4467,
+      "step": 410
+    },
+    {
+      "epoch": 1.2321495327102805,
+      "grad_norm": 0.5700398683547974,
+      "learning_rate": 6.846457126640097e-06,
+      "loss": 0.464,
+      "step": 412
+    },
+    {
+      "epoch": 1.2381308411214953,
+      "grad_norm": 0.5757970809936523,
+      "learning_rate": 6.8157175240269495e-06,
+      "loss": 0.4471,
+      "step": 414
+    },
+    {
+      "epoch": 1.2441121495327103,
+      "grad_norm": 0.5315657258033752,
+      "learning_rate": 6.784898662994235e-06,
+      "loss": 0.441,
+      "step": 416
+    },
+    {
+      "epoch": 1.2500934579439251,
+      "grad_norm": 0.5489743947982788,
+      "learning_rate": 6.75400188882512e-06,
+      "loss": 0.4468,
+      "step": 418
+    },
+    {
+      "epoch": 1.2560747663551401,
+      "grad_norm": 0.5474492311477661,
+      "learning_rate": 6.723028550203779e-06,
+      "loss": 0.4179,
+      "step": 420
+    },
+    {
+      "epoch": 1.2620560747663552,
+      "grad_norm": 2.438246965408325,
+      "learning_rate": 6.691979999156521e-06,
+      "loss": 0.4669,
+      "step": 422
+    },
+    {
+      "epoch": 1.2680373831775702,
+      "grad_norm": 0.5756619572639465,
+      "learning_rate": 6.660857590992781e-06,
+      "loss": 0.4535,
+      "step": 424
+    },
+    {
+      "epoch": 1.274018691588785,
+      "grad_norm": 0.578809380531311,
+      "learning_rate": 6.629662684245949e-06,
+      "loss": 0.4345,
+      "step": 426
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 0.5422917008399963,
+      "learning_rate": 6.598396640614072e-06,
+      "loss": 0.4144,
+      "step": 428
+    },
+    {
+      "epoch": 1.2859813084112148,
+      "grad_norm": 0.5585193634033203,
+      "learning_rate": 6.567060824900414e-06,
+      "loss": 0.4472,
+      "step": 430
+    },
+    {
+      "epoch": 1.2919626168224299,
+      "grad_norm": 0.5663349628448486,
+      "learning_rate": 6.535656604953884e-06,
+      "loss": 0.4552,
+      "step": 432
+    },
+    {
+      "epoch": 1.2979439252336449,
+      "grad_norm": 0.5570898652076721,
+      "learning_rate": 6.504185351609318e-06,
+      "loss": 0.4481,
+      "step": 434
+    },
+    {
+      "epoch": 1.30392523364486,
+      "grad_norm": 0.5667449235916138,
+      "learning_rate": 6.4726484386276525e-06,
+      "loss": 0.424,
+      "step": 436
+    },
+    {
+      "epoch": 1.3099065420560747,
+      "grad_norm": 0.573315441608429,
+      "learning_rate": 6.441047242635947e-06,
+      "loss": 0.4297,
+      "step": 438
+    },
+    {
+      "epoch": 1.3158878504672897,
+      "grad_norm": 0.5521847009658813,
+      "learning_rate": 6.409383143067305e-06,
+      "loss": 0.4459,
+      "step": 440
+    },
+    {
+      "epoch": 1.3218691588785045,
+      "grad_norm": 0.5647687911987305,
+      "learning_rate": 6.377657522100644e-06,
+      "loss": 0.4444,
+      "step": 442
+    },
+    {
+      "epoch": 1.3278504672897196,
+      "grad_norm": 0.5553613305091858,
+      "learning_rate": 6.3458717646003746e-06,
+      "loss": 0.4457,
+      "step": 444
+    },
+    {
+      "epoch": 1.3338317757009346,
+      "grad_norm": 0.9647654891014099,
+      "learning_rate": 6.3140272580559436e-06,
+      "loss": 0.4511,
+      "step": 446
+    },
+    {
+      "epoch": 1.3398130841121496,
+      "grad_norm": 0.5839756727218628,
+      "learning_rate": 6.2821253925212736e-06,
+      "loss": 0.4478,
+      "step": 448
+    },
+    {
+      "epoch": 1.3457943925233644,
+      "grad_norm": 0.56339031457901,
+      "learning_rate": 6.2501675605540755e-06,
+      "loss": 0.4374,
+      "step": 450
+    },
+    {
+      "epoch": 1.3517757009345794,
+      "grad_norm": 0.5671836733818054,
+      "learning_rate": 6.2181551571550704e-06,
+      "loss": 0.4378,
+      "step": 452
+    },
+    {
+      "epoch": 1.3577570093457945,
+      "grad_norm": 0.5433515310287476,
+      "learning_rate": 6.186089579707091e-06,
+      "loss": 0.4429,
+      "step": 454
+    },
+    {
+      "epoch": 1.3637383177570093,
+      "grad_norm": 0.5540264844894409,
+      "learning_rate": 6.153972227914089e-06,
+      "loss": 0.4175,
+      "step": 456
+    },
+    {
+      "epoch": 1.3697196261682243,
+      "grad_norm": 0.5408939123153687,
+      "learning_rate": 6.121804503740034e-06,
+      "loss": 0.4393,
+      "step": 458
+    },
+    {
+      "epoch": 1.3757009345794393,
+      "grad_norm": 0.5199695229530334,
+      "learning_rate": 6.0895878113477105e-06,
+      "loss": 0.4152,
+      "step": 460
+    },
+    {
+      "epoch": 1.3816822429906543,
+      "grad_norm": 0.5756475329399109,
+      "learning_rate": 6.0573235570374315e-06,
+      "loss": 0.4624,
+      "step": 462
+    },
+    {
+      "epoch": 1.3876635514018691,
+      "grad_norm": 0.5712220072746277,
+      "learning_rate": 6.02501314918565e-06,
+      "loss": 0.4304,
+      "step": 464
+    },
+    {
+      "epoch": 1.3936448598130842,
+      "grad_norm": 0.5634981393814087,
+      "learning_rate": 5.99265799818348e-06,
+      "loss": 0.4364,
+      "step": 466
+    },
+    {
+      "epoch": 1.399626168224299,
+      "grad_norm": 0.5494462847709656,
+      "learning_rate": 5.960259516375134e-06,
+      "loss": 0.4405,
+      "step": 468
+    },
+    {
+      "epoch": 1.405607476635514,
+      "grad_norm": 0.5732921361923218,
+      "learning_rate": 5.927819117996267e-06,
+      "loss": 0.4396,
+      "step": 470
+    },
+    {
+      "epoch": 1.411588785046729,
+      "grad_norm": 0.5824448466300964,
+      "learning_rate": 5.895338219112246e-06,
+      "loss": 0.4592,
+      "step": 472
+    },
+    {
+      "epoch": 1.417570093457944,
+      "grad_norm": 0.5585994124412537,
+      "learning_rate": 5.862818237556344e-06,
+      "loss": 0.4426,
+      "step": 474
+    },
+    {
+      "epoch": 1.4235514018691589,
+      "grad_norm": 0.5504030585289001,
+      "learning_rate": 5.8302605928678345e-06,
+      "loss": 0.435,
+      "step": 476
+    },
+    {
+      "epoch": 1.4295327102803739,
+      "grad_norm": 0.5663231611251831,
+      "learning_rate": 5.797666706230042e-06,
+      "loss": 0.4392,
+      "step": 478
+    },
+    {
+      "epoch": 1.4355140186915887,
+      "grad_norm": 0.5592628121376038,
+      "learning_rate": 5.765038000408295e-06,
+      "loss": 0.4401,
+      "step": 480
+    },
+    {
+      "epoch": 1.4414953271028037,
+      "grad_norm": 0.5621228218078613,
+      "learning_rate": 5.7323758996878275e-06,
+      "loss": 0.4361,
+      "step": 482
+    },
+    {
+      "epoch": 1.4474766355140187,
+      "grad_norm": 0.5564706921577454,
+      "learning_rate": 5.699681829811602e-06,
+      "loss": 0.4514,
+      "step": 484
+    },
+    {
+      "epoch": 1.4534579439252338,
+      "grad_norm": 0.5423418879508972,
+      "learning_rate": 5.666957217918076e-06,
+      "loss": 0.4311,
+      "step": 486
+    },
+    {
+      "epoch": 1.4594392523364486,
+      "grad_norm": 0.5552216172218323,
+      "learning_rate": 5.634203492478909e-06,
+      "loss": 0.4402,
+      "step": 488
+    },
+    {
+      "epoch": 1.4654205607476636,
+      "grad_norm": 0.5619707703590393,
+      "learning_rate": 5.601422083236596e-06,
+      "loss": 0.4436,
+      "step": 490
+    },
+    {
+      "epoch": 1.4714018691588784,
+      "grad_norm": 0.5552605390548706,
+      "learning_rate": 5.568614421142078e-06,
+      "loss": 0.4222,
+      "step": 492
+    },
+    {
+      "epoch": 1.4773831775700934,
+      "grad_norm": 0.6014040112495422,
+      "learning_rate": 5.535781938292255e-06,
+      "loss": 0.4384,
+      "step": 494
+    },
+    {
+      "epoch": 1.4833644859813084,
+      "grad_norm": 0.5808812379837036,
+      "learning_rate": 5.502926067867495e-06,
+      "loss": 0.4543,
+      "step": 496
+    },
+    {
+      "epoch": 1.4893457943925235,
+      "grad_norm": 0.6073839068412781,
+      "learning_rate": 5.470048244069055e-06,
+      "loss": 0.4742,
+      "step": 498
+    },
+    {
+      "epoch": 1.4953271028037383,
+      "grad_norm": 0.5317693948745728,
+      "learning_rate": 5.437149902056494e-06,
+      "loss": 0.4166,
+      "step": 500
+    },
+    {
+      "epoch": 1.5013084112149533,
+      "grad_norm": 0.5628969073295593,
+      "learning_rate": 5.4042324778850095e-06,
+      "loss": 0.4621,
+      "step": 502
+    },
+    {
+      "epoch": 1.507289719626168,
+      "grad_norm": 0.5735781788825989,
+      "learning_rate": 5.371297408442765e-06,
+      "loss": 0.4515,
+      "step": 504
+    },
+    {
+      "epoch": 1.5132710280373831,
+      "grad_norm": 0.5652995109558105,
+      "learning_rate": 5.338346131388161e-06,
+      "loss": 0.4269,
+      "step": 506
+    },
+    {
+      "epoch": 1.5192523364485981,
+      "grad_norm": 0.5484395623207092,
+      "learning_rate": 5.305380085087085e-06,
+      "loss": 0.4488,
+      "step": 508
+    },
+    {
+      "epoch": 1.5252336448598132,
+      "grad_norm": 0.5202231407165527,
+      "learning_rate": 5.272400708550114e-06,
+      "loss": 0.4339,
+      "step": 510
+    },
+    {
+      "epoch": 1.5312149532710282,
+      "grad_norm": 8.480402946472168,
+      "learning_rate": 5.239409441369714e-06,
+      "loss": 0.4648,
+      "step": 512
+    },
+    {
+      "epoch": 1.537196261682243,
+      "grad_norm": 0.5879077315330505,
+      "learning_rate": 5.2064077236573865e-06,
+      "loss": 0.4502,
+      "step": 514
+    },
+    {
+      "epoch": 1.5431775700934578,
+      "grad_norm": 0.5753892660140991,
+      "learning_rate": 5.173396995980818e-06,
+      "loss": 0.466,
+      "step": 516
+    },
+    {
+      "epoch": 1.5491588785046728,
+      "grad_norm": 0.5262240171432495,
+      "learning_rate": 5.1403786993009905e-06,
+      "loss": 0.4413,
+      "step": 518
+    },
+    {
+      "epoch": 1.5551401869158878,
+      "grad_norm": 1.5011541843414307,
+      "learning_rate": 5.107354274909281e-06,
+      "loss": 0.4541,
+      "step": 520
+    },
+    {
+      "epoch": 1.5611214953271029,
+      "grad_norm": 0.5343008637428284,
+      "learning_rate": 5.074325164364549e-06,
+      "loss": 0.4307,
+      "step": 522
+    },
+    {
+      "epoch": 1.567102803738318,
+      "grad_norm": 0.5578626990318298,
+      "learning_rate": 5.04129280943021e-06,
+      "loss": 0.4381,
+      "step": 524
+    },
+    {
+      "epoch": 1.5730841121495327,
+      "grad_norm": 0.5335695743560791,
+      "learning_rate": 5.008258652011303e-06,
+      "loss": 0.4215,
+      "step": 526
+    },
+    {
+      "epoch": 1.5790654205607475,
+      "grad_norm": 0.5358263850212097,
+      "learning_rate": 4.975224134091551e-06,
+      "loss": 0.4449,
+      "step": 528
+    },
+    {
+      "epoch": 1.5850467289719625,
+      "grad_norm": 0.559319257736206,
+      "learning_rate": 4.942190697670407e-06,
+      "loss": 0.4128,
+      "step": 530
+    },
+    {
+      "epoch": 1.5910280373831776,
+      "grad_norm": 0.5156571865081787,
+      "learning_rate": 4.909159784700118e-06,
+      "loss": 0.4193,
+      "step": 532
+    },
+    {
+      "epoch": 1.5970093457943926,
+      "grad_norm": 0.5598333477973938,
+      "learning_rate": 4.876132837022778e-06,
+      "loss": 0.4569,
+      "step": 534
+    },
+    {
+      "epoch": 1.6029906542056076,
+      "grad_norm": 0.5344600677490234,
+      "learning_rate": 4.843111296307392e-06,
+      "loss": 0.4216,
+      "step": 536
+    },
+    {
+      "epoch": 1.6089719626168224,
+      "grad_norm": 0.538439154624939,
+      "learning_rate": 4.810096603986948e-06,
+      "loss": 0.4546,
+      "step": 538
+    },
+    {
+      "epoch": 1.6149532710280374,
+      "grad_norm": 0.5335719585418701,
+      "learning_rate": 4.777090201195486e-06,
+      "loss": 0.4466,
+      "step": 540
+    },
+    {
+      "epoch": 1.6209345794392522,
+      "grad_norm": 0.5561640858650208,
+      "learning_rate": 4.7440935287052e-06,
+      "loss": 0.446,
+      "step": 542
+    },
+    {
+      "epoch": 1.6269158878504673,
+      "grad_norm": 0.5510598421096802,
+      "learning_rate": 4.711108026863542e-06,
+      "loss": 0.4424,
+      "step": 544
+    },
+    {
+      "epoch": 1.6328971962616823,
+      "grad_norm": 0.6612152457237244,
+      "learning_rate": 4.6781351355303555e-06,
+      "loss": 0.4039,
+      "step": 546
+    },
+    {
+      "epoch": 1.6388785046728973,
+      "grad_norm": 0.5314252972602844,
+      "learning_rate": 4.645176294015015e-06,
+      "loss": 0.4409,
+      "step": 548
+    },
+    {
+      "epoch": 1.644859813084112,
+      "grad_norm": 0.5514910221099854,
+      "learning_rate": 4.612232941013604e-06,
+      "loss": 0.4442,
+      "step": 550
+    },
+    {
+      "epoch": 1.6508411214953271,
+      "grad_norm": 0.5438050627708435,
+      "learning_rate": 4.579306514546107e-06,
+      "loss": 0.4168,
+      "step": 552
+    },
+    {
+      "epoch": 1.656822429906542,
+      "grad_norm": 0.5547947287559509,
+      "learning_rate": 4.546398451893654e-06,
+      "loss": 0.4398,
+      "step": 554
+    },
+    {
+      "epoch": 1.662803738317757,
+      "grad_norm": 0.5689917802810669,
+      "learning_rate": 4.513510189535762e-06,
+      "loss": 0.4353,
+      "step": 556
+    },
+    {
+      "epoch": 1.668785046728972,
+      "grad_norm": 0.5358143448829651,
+      "learning_rate": 4.480643163087644e-06,
+      "loss": 0.4177,
+      "step": 558
+    },
+    {
+      "epoch": 1.674766355140187,
+      "grad_norm": 0.5444421172142029,
+      "learning_rate": 4.447798807237535e-06,
+      "loss": 0.4397,
+      "step": 560
+    },
+    {
+      "epoch": 1.6807476635514018,
+      "grad_norm": 0.5441679358482361,
+      "learning_rate": 4.414978555684069e-06,
+      "loss": 0.4165,
+      "step": 562
+    },
+    {
+      "epoch": 1.6867289719626168,
+      "grad_norm": 0.5753957033157349,
+      "learning_rate": 4.382183841073698e-06,
+      "loss": 0.4486,
+      "step": 564
+    },
+    {
+      "epoch": 1.6927102803738316,
+      "grad_norm": 0.530971109867096,
+      "learning_rate": 4.349416094938156e-06,
+      "loss": 0.4208,
+      "step": 566
+    },
+    {
+      "epoch": 1.6986915887850467,
+      "grad_norm": 0.974751889705658,
+      "learning_rate": 4.316676747631963e-06,
+      "loss": 0.4448,
+      "step": 568
+    },
+    {
+      "epoch": 1.7046728971962617,
+      "grad_norm": 0.5555000305175781,
+      "learning_rate": 4.283967228269993e-06,
+      "loss": 0.4302,
+      "step": 570
+    },
+    {
+      "epoch": 1.7106542056074767,
+      "grad_norm": 0.5458599925041199,
+      "learning_rate": 4.2512889646650975e-06,
+      "loss": 0.4208,
+      "step": 572
+    },
+    {
+      "epoch": 1.7166355140186917,
+      "grad_norm": 0.5513786673545837,
+      "learning_rate": 4.218643383265771e-06,
+      "loss": 0.4337,
+      "step": 574
+    },
+    {
+      "epoch": 1.7226168224299065,
+      "grad_norm": 0.5598821043968201,
+      "learning_rate": 4.186031909093884e-06,
+      "loss": 0.417,
+      "step": 576
+    },
+    {
+      "epoch": 1.7285981308411213,
+      "grad_norm": 0.5501388907432556,
+      "learning_rate": 4.153455965682489e-06,
+      "loss": 0.4397,
+      "step": 578
+    },
+    {
+      "epoch": 1.7345794392523364,
+      "grad_norm": 0.5336859822273254,
+      "learning_rate": 4.120916975013668e-06,
+      "loss": 0.4527,
+      "step": 580
+    },
+    {
+      "epoch": 1.7405607476635514,
+      "grad_norm": 0.5666311383247375,
+      "learning_rate": 4.088416357456471e-06,
+      "loss": 0.4538,
+      "step": 582
+    },
+    {
+      "epoch": 1.7465420560747664,
+      "grad_norm": 0.5301072597503662,
+      "learning_rate": 4.055955531704912e-06,
+      "loss": 0.4289,
+      "step": 584
+    },
+    {
+      "epoch": 1.7525233644859814,
+      "grad_norm": 0.5418647527694702,
+      "learning_rate": 4.023535914716043e-06,
+      "loss": 0.4374,
+      "step": 586
+    },
+    {
+      "epoch": 1.7585046728971963,
+      "grad_norm": 0.5535775423049927,
+      "learning_rate": 3.991158921648096e-06,
+      "loss": 0.4409,
+      "step": 588
+    },
+    {
+      "epoch": 1.764485981308411,
+      "grad_norm": 0.5371606349945068,
+      "learning_rate": 3.95882596579872e-06,
+      "loss": 0.4471,
+      "step": 590
+    },
+    {
+      "epoch": 1.770467289719626,
+      "grad_norm": 0.5358505249023438,
+      "learning_rate": 3.926538458543275e-06,
+      "loss": 0.4134,
+      "step": 592
+    },
+    {
+      "epoch": 1.776448598130841,
+      "grad_norm": 0.5423475503921509,
+      "learning_rate": 3.894297809273237e-06,
+      "loss": 0.4317,
+      "step": 594
+    },
+    {
+      "epoch": 1.7824299065420561,
+      "grad_norm": 0.5343226790428162,
+      "learning_rate": 3.862105425334665e-06,
+      "loss": 0.4327,
+      "step": 596
+    },
+    {
+      "epoch": 1.7884112149532712,
+      "grad_norm": 0.5356937050819397,
+      "learning_rate": 3.829962711966779e-06,
+      "loss": 0.4229,
+      "step": 598
+    },
+    {
+      "epoch": 1.794392523364486,
+      "grad_norm": 0.5763903856277466,
+      "learning_rate": 3.7978710722406113e-06,
+      "loss": 0.4632,
+      "step": 600
+    },
+    {
+      "epoch": 1.800373831775701,
+      "grad_norm": 0.5528432726860046,
+      "learning_rate": 3.765831906997765e-06,
+      "loss": 0.4175,
+      "step": 602
+    },
+    {
+      "epoch": 1.8063551401869158,
+      "grad_norm": 0.5658871531486511,
+      "learning_rate": 3.7338466147892603e-06,
+      "loss": 0.4276,
+      "step": 604
+    },
+    {
+      "epoch": 1.8123364485981308,
+      "grad_norm": 0.5515477061271667,
+      "learning_rate": 3.7019165918144974e-06,
+      "loss": 0.4487,
+      "step": 606
+    },
+    {
+      "epoch": 1.8183177570093458,
+      "grad_norm": 0.5468549728393555,
+      "learning_rate": 3.670043231860297e-06,
+      "loss": 0.4489,
+      "step": 608
+    },
+    {
+      "epoch": 1.8242990654205609,
+      "grad_norm": 0.5574175715446472,
+      "learning_rate": 3.6382279262400665e-06,
+      "loss": 0.4394,
+      "step": 610
+    },
+    {
+      "epoch": 1.8302803738317757,
+      "grad_norm": 0.5626105666160583,
+      "learning_rate": 3.6064720637330673e-06,
+      "loss": 0.4485,
+      "step": 612
+    },
+    {
+      "epoch": 1.8362616822429907,
+      "grad_norm": 0.5468553900718689,
+      "learning_rate": 3.574777030523791e-06,
+      "loss": 0.449,
+      "step": 614
+    },
+    {
+      "epoch": 1.8422429906542055,
+      "grad_norm": 0.8080570697784424,
+      "learning_rate": 3.5431442101414537e-06,
+      "loss": 0.4405,
+      "step": 616
+    },
+    {
+      "epoch": 1.8482242990654205,
+      "grad_norm": 0.5403231382369995,
+      "learning_rate": 3.511574983399599e-06,
+      "loss": 0.4324,
+      "step": 618
+    },
+    {
+      "epoch": 1.8542056074766355,
+      "grad_norm": 0.5402336120605469,
+      "learning_rate": 3.4800707283358253e-06,
+      "loss": 0.4438,
+      "step": 620
+    },
+    {
+      "epoch": 1.8601869158878506,
+      "grad_norm": 0.5512120127677917,
+      "learning_rate": 3.448632820151638e-06,
+      "loss": 0.4449,
+      "step": 622
+    },
+    {
+      "epoch": 1.8661682242990654,
+      "grad_norm": 0.5519320368766785,
+      "learning_rate": 3.417262631152409e-06,
+      "loss": 0.421,
+      "step": 624
+    },
+    {
+      "epoch": 1.8721495327102804,
+      "grad_norm": 0.5409705638885498,
+      "learning_rate": 3.3859615306874827e-06,
+      "loss": 0.4072,
+      "step": 626
+    },
+    {
+      "epoch": 1.8781308411214952,
+      "grad_norm": 0.5319204926490784,
+      "learning_rate": 3.3547308850904017e-06,
+      "loss": 0.4244,
+      "step": 628
+    },
+    {
+      "epoch": 1.8841121495327102,
+      "grad_norm": 0.5436004400253296,
+      "learning_rate": 3.32357205761926e-06,
+      "loss": 0.4149,
+      "step": 630
+    },
+    {
+      "epoch": 1.8900934579439252,
+      "grad_norm": 0.5732548236846924,
+      "learning_rate": 3.2924864083971975e-06,
+      "loss": 0.4502,
+      "step": 632
+    },
+    {
+      "epoch": 1.8960747663551403,
+      "grad_norm": 0.5404754877090454,
+      "learning_rate": 3.261475294353028e-06,
+      "loss": 0.433,
+      "step": 634
+    },
+    {
+      "epoch": 1.9020560747663553,
+      "grad_norm": 0.5527997016906738,
+      "learning_rate": 3.2305400691620126e-06,
+      "loss": 0.4411,
+      "step": 636
+    },
+    {
+      "epoch": 1.90803738317757,
+      "grad_norm": 0.5835505723953247,
+      "learning_rate": 3.1996820831867593e-06,
+      "loss": 0.4492,
+      "step": 638
+    },
+    {
+      "epoch": 1.914018691588785,
+      "grad_norm": 0.5378055572509766,
+      "learning_rate": 3.1689026834182923e-06,
+      "loss": 0.4444,
+      "step": 640
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 0.5413604378700256,
+      "learning_rate": 3.1382032134172395e-06,
+      "loss": 0.4355,
+      "step": 642
+    },
+    {
+      "epoch": 1.925981308411215,
+      "grad_norm": 0.5299492478370667,
+      "learning_rate": 3.107585013255194e-06,
+      "loss": 0.4273,
+      "step": 644
+    },
+    {
+      "epoch": 1.93196261682243,
+      "grad_norm": 0.5561743974685669,
+      "learning_rate": 3.077049419456212e-06,
+      "loss": 0.4507,
+      "step": 646
+    },
+    {
+      "epoch": 1.937943925233645,
+      "grad_norm": 0.5475538372993469,
+      "learning_rate": 3.0465977649384813e-06,
+      "loss": 0.4296,
+      "step": 648
+    },
+    {
+      "epoch": 1.9439252336448598,
+      "grad_norm": 0.5660334825515747,
+      "learning_rate": 3.016231378956126e-06,
+      "loss": 0.4339,
+      "step": 650
+    },
+    {
+      "epoch": 1.9499065420560746,
+      "grad_norm": 0.5578892827033997,
+      "learning_rate": 2.9859515870411905e-06,
+      "loss": 0.4536,
+      "step": 652
+    },
+    {
+      "epoch": 1.9558878504672896,
+      "grad_norm": 0.5542095899581909,
+      "learning_rate": 2.955759710945773e-06,
+      "loss": 0.4562,
+      "step": 654
+    },
+    {
+      "epoch": 1.9618691588785047,
+      "grad_norm": 0.551667332649231,
+      "learning_rate": 2.92565706858434e-06,
+      "loss": 0.4401,
+      "step": 656
+    },
+    {
+      "epoch": 1.9678504672897197,
+      "grad_norm": 0.5226486921310425,
+      "learning_rate": 2.8956449739761816e-06,
+      "loss": 0.4493,
+      "step": 658
+    },
+    {
+      "epoch": 1.9738317757009347,
+      "grad_norm": 0.5326710939407349,
+      "learning_rate": 2.865724737188067e-06,
+      "loss": 0.4356,
+      "step": 660
+    },
+    {
+      "epoch": 1.9798130841121495,
+      "grad_norm": 0.5447040796279907,
+      "learning_rate": 2.8358976642770486e-06,
+      "loss": 0.453,
+      "step": 662
+    },
+    {
+      "epoch": 1.9857943925233645,
+      "grad_norm": 0.5460859537124634,
+      "learning_rate": 2.8061650572334572e-06,
+      "loss": 0.4134,
+      "step": 664
+    },
+    {
+      "epoch": 1.9917757009345793,
+      "grad_norm": 0.5193696022033691,
+      "learning_rate": 2.776528213924068e-06,
+      "loss": 0.4277,
+      "step": 666
+    },
+    {
+      "epoch": 1.9977570093457944,
+      "grad_norm": 0.532832682132721,
+      "learning_rate": 2.7469884280354404e-06,
+      "loss": 0.4386,
+      "step": 668
+    },
+    {
+      "epoch": 2.0037383177570094,
+      "grad_norm": 0.9088693261146545,
+      "learning_rate": 2.7175469890174556e-06,
+      "loss": 0.5557,
+      "step": 670
+    },
+    {
+      "epoch": 2.0097196261682244,
+      "grad_norm": 0.8152005076408386,
+      "learning_rate": 2.688205182027026e-06,
+      "loss": 0.3505,
+      "step": 672
+    },
+    {
+      "epoch": 2.0157009345794394,
+      "grad_norm": 0.6255513429641724,
+      "learning_rate": 2.6589642878719967e-06,
+      "loss": 0.3743,
+      "step": 674
+    },
+    {
+      "epoch": 2.021682242990654,
+      "grad_norm": 0.8579421639442444,
+      "learning_rate": 2.6298255829552393e-06,
+      "loss": 0.3673,
+      "step": 676
+    },
+    {
+      "epoch": 2.027663551401869,
+      "grad_norm": 0.8957614302635193,
+      "learning_rate": 2.600790339218926e-06,
+      "loss": 0.3609,
+      "step": 678
+    },
+    {
+      "epoch": 2.033644859813084,
+      "grad_norm": 0.6624680757522583,
+      "learning_rate": 2.5718598240890213e-06,
+      "loss": 0.3719,
+      "step": 680
+    },
+    {
+      "epoch": 2.039626168224299,
+      "grad_norm": 0.7025406360626221,
+      "learning_rate": 2.543035300419951e-06,
+      "loss": 0.3665,
+      "step": 682
+    },
+    {
+      "epoch": 2.045607476635514,
+      "grad_norm": 0.6330462694168091,
+      "learning_rate": 2.514318026439469e-06,
+      "loss": 0.3658,
+      "step": 684
+    },
+    {
+      "epoch": 2.051588785046729,
+      "grad_norm": 0.6362036466598511,
+      "learning_rate": 2.4857092556937505e-06,
+      "loss": 0.3686,
+      "step": 686
+    },
+    {
+      "epoch": 2.0575700934579437,
+      "grad_norm": 0.6507929563522339,
+      "learning_rate": 2.4572102369926547e-06,
+      "loss": 0.3499,
+      "step": 688
+    },
+    {
+      "epoch": 2.0635514018691588,
+      "grad_norm": 0.5907099843025208,
+      "learning_rate": 2.428822214355235e-06,
+      "loss": 0.3603,
+      "step": 690
+    },
+    {
+      "epoch": 2.0695327102803738,
+      "grad_norm": 0.6001895070075989,
+      "learning_rate": 2.400546426955408e-06,
+      "loss": 0.362,
+      "step": 692
+    },
+    {
+      "epoch": 2.075514018691589,
+      "grad_norm": 0.5974778532981873,
+      "learning_rate": 2.3723841090678878e-06,
+      "loss": 0.3601,
+      "step": 694
+    },
+    {
+      "epoch": 2.081495327102804,
+      "grad_norm": 0.5870632529258728,
+      "learning_rate": 2.344336490014295e-06,
+      "loss": 0.3564,
+      "step": 696
+    },
+    {
+      "epoch": 2.087476635514019,
+      "grad_norm": 0.5829831957817078,
+      "learning_rate": 2.316404794109493e-06,
+      "loss": 0.3437,
+      "step": 698
+    },
+    {
+      "epoch": 2.0934579439252334,
+      "grad_norm": 0.6477536559104919,
+      "learning_rate": 2.288590240608155e-06,
+      "loss": 0.3707,
+      "step": 700
+    },
+    {
+      "epoch": 2.0994392523364485,
+      "grad_norm": 0.6187012195587158,
+      "learning_rate": 2.260894043651537e-06,
+      "loss": 0.3932,
+      "step": 702
+    },
+    {
+      "epoch": 2.1054205607476635,
+      "grad_norm": 0.594266951084137,
+      "learning_rate": 2.233317412214473e-06,
+      "loss": 0.3768,
+      "step": 704
+    },
+    {
+      "epoch": 2.1114018691588785,
+      "grad_norm": 0.5813930034637451,
+      "learning_rate": 2.205861550052612e-06,
+      "loss": 0.3645,
+      "step": 706
+    },
+    {
+      "epoch": 2.1173831775700935,
+      "grad_norm": 0.5965720415115356,
+      "learning_rate": 2.178527655649868e-06,
+      "loss": 0.3534,
+      "step": 708
+    },
+    {
+      "epoch": 2.1233644859813086,
+      "grad_norm": 0.6631720066070557,
+      "learning_rate": 2.1513169221661047e-06,
+      "loss": 0.347,
+      "step": 710
+    },
+    {
+      "epoch": 2.129345794392523,
+      "grad_norm": 2.115356683731079,
+      "learning_rate": 2.124230537385047e-06,
+      "loss": 0.3551,
+      "step": 712
+    },
+    {
+      "epoch": 2.135327102803738,
+      "grad_norm": 0.6132083535194397,
+      "learning_rate": 2.097269683662444e-06,
+      "loss": 0.3534,
+      "step": 714
+    },
+    {
+      "epoch": 2.141308411214953,
+      "grad_norm": 0.5820258259773254,
+      "learning_rate": 2.070435537874451e-06,
+      "loss": 0.3503,
+      "step": 716
+    },
+    {
+      "epoch": 2.147289719626168,
+      "grad_norm": 0.59654700756073,
+      "learning_rate": 2.0437292713662537e-06,
+      "loss": 0.3497,
+      "step": 718
+    },
+    {
+      "epoch": 2.1532710280373832,
+      "grad_norm": 0.5738710165023804,
+      "learning_rate": 2.0171520499009457e-06,
+      "loss": 0.3507,
+      "step": 720
+    },
+    {
+      "epoch": 2.1592523364485983,
+      "grad_norm": 0.5718917846679688,
+      "learning_rate": 1.990705033608637e-06,
+      "loss": 0.3404,
+      "step": 722
+    },
+    {
+      "epoch": 2.1652336448598133,
+      "grad_norm": 0.5919547080993652,
+      "learning_rate": 1.9643893769358124e-06,
+      "loss": 0.3506,
+      "step": 724
+    },
+    {
+      "epoch": 2.171214953271028,
+      "grad_norm": 0.5977197289466858,
+      "learning_rate": 1.9382062285949416e-06,
+      "loss": 0.3569,
+      "step": 726
+    },
+    {
+      "epoch": 2.177196261682243,
+      "grad_norm": 0.5944223403930664,
+      "learning_rate": 1.9121567315143285e-06,
+      "loss": 0.3632,
+      "step": 728
+    },
+    {
+      "epoch": 2.183177570093458,
+      "grad_norm": 0.6034772992134094,
+      "learning_rate": 1.8862420227882338e-06,
+      "loss": 0.3404,
+      "step": 730
+    },
+    {
+      "epoch": 2.189158878504673,
+      "grad_norm": 0.603488564491272,
+      "learning_rate": 1.8604632336272249e-06,
+      "loss": 0.3764,
+      "step": 732
+    },
+    {
+      "epoch": 2.195140186915888,
+      "grad_norm": 0.5720504522323608,
+      "learning_rate": 1.8348214893088101e-06,
+      "loss": 0.3556,
+      "step": 734
+    },
+    {
+      "epoch": 2.201121495327103,
+      "grad_norm": 0.6171439290046692,
+      "learning_rate": 1.809317909128312e-06,
+      "loss": 0.3598,
+      "step": 736
+    },
+    {
+      "epoch": 2.2071028037383176,
+      "grad_norm": 0.5888059735298157,
+      "learning_rate": 1.783953606350005e-06,
+      "loss": 0.3791,
+      "step": 738
+    },
+    {
+      "epoch": 2.2130841121495326,
+      "grad_norm": 0.5837867259979248,
+      "learning_rate": 1.7587296881585308e-06,
+      "loss": 0.3601,
+      "step": 740
+    },
+    {
+      "epoch": 2.2190654205607476,
+      "grad_norm": 0.5914073586463928,
+      "learning_rate": 1.7336472556105594e-06,
+      "loss": 0.3421,
+      "step": 742
+    },
+    {
+      "epoch": 2.2250467289719627,
+      "grad_norm": 0.6119322180747986,
+      "learning_rate": 1.7087074035867284e-06,
+      "loss": 0.3693,
+      "step": 744
+    },
+    {
+      "epoch": 2.2310280373831777,
+      "grad_norm": 0.6087052226066589,
+      "learning_rate": 1.6839112207438558e-06,
+      "loss": 0.375,
+      "step": 746
+    },
+    {
+      "epoch": 2.2370093457943927,
+      "grad_norm": 0.6128002405166626,
+      "learning_rate": 1.6592597894674056e-06,
+      "loss": 0.3657,
+      "step": 748
+    },
+    {
+      "epoch": 2.2429906542056073,
+      "grad_norm": 0.5853055715560913,
+      "learning_rate": 1.634754185824256e-06,
+      "loss": 0.3738,
+      "step": 750
+    },
+    {
+      "epoch": 2.2489719626168223,
+      "grad_norm": 0.6305050849914551,
+      "learning_rate": 1.6103954795157188e-06,
+      "loss": 0.3716,
+      "step": 752
+    },
+    {
+      "epoch": 2.2549532710280373,
+      "grad_norm": 0.6319517493247986,
+      "learning_rate": 1.586184733830843e-06,
+      "loss": 0.366,
+      "step": 754
+    },
+    {
+      "epoch": 2.2609345794392524,
+      "grad_norm": 0.5690290331840515,
+      "learning_rate": 1.562123005600009e-06,
+      "loss": 0.3577,
+      "step": 756
+    },
+    {
+      "epoch": 2.2669158878504674,
+      "grad_norm": 0.5914164185523987,
+      "learning_rate": 1.5382113451487918e-06,
+      "loss": 0.3722,
+      "step": 758
+    },
+    {
+      "epoch": 2.2728971962616824,
+      "grad_norm": 0.5985538363456726,
+      "learning_rate": 1.5144507962521144e-06,
+      "loss": 0.368,
+      "step": 760
+    },
+    {
+      "epoch": 2.278878504672897,
+      "grad_norm": 0.6006935834884644,
+      "learning_rate": 1.4908423960886808e-06,
+      "loss": 0.3399,
+      "step": 762
+    },
+    {
+      "epoch": 2.284859813084112,
+      "grad_norm": 0.6002554893493652,
+      "learning_rate": 1.4673871751957102e-06,
+      "loss": 0.3601,
+      "step": 764
+    },
+    {
+      "epoch": 2.290841121495327,
+      "grad_norm": 0.5984997749328613,
+      "learning_rate": 1.4440861574239484e-06,
+      "loss": 0.3665,
+      "step": 766
+    },
+    {
+      "epoch": 2.296822429906542,
+      "grad_norm": 0.5855640172958374,
+      "learning_rate": 1.4209403598929711e-06,
+      "loss": 0.3515,
+      "step": 768
+    },
+    {
+      "epoch": 2.302803738317757,
+      "grad_norm": 0.6179247498512268,
+      "learning_rate": 1.3979507929467944e-06,
+      "loss": 0.3654,
+      "step": 770
+    },
+    {
+      "epoch": 2.308785046728972,
+      "grad_norm": 0.6082316040992737,
+      "learning_rate": 1.3751184601097662e-06,
+      "loss": 0.3475,
+      "step": 772
+    },
+    {
+      "epoch": 2.314766355140187,
+      "grad_norm": 0.625310480594635,
+      "learning_rate": 1.3524443580427565e-06,
+      "loss": 0.3754,
+      "step": 774
+    },
+    {
+      "epoch": 2.3207476635514017,
+      "grad_norm": 0.5993301272392273,
+      "learning_rate": 1.329929476499669e-06,
+      "loss": 0.3411,
+      "step": 776
+    },
+    {
+      "epoch": 2.3267289719626167,
+      "grad_norm": 0.6278132796287537,
+      "learning_rate": 1.3075747982842135e-06,
+      "loss": 0.3703,
+      "step": 778
+    },
+    {
+      "epoch": 2.3327102803738318,
+      "grad_norm": 0.590921938419342,
+      "learning_rate": 1.285381299207026e-06,
+      "loss": 0.37,
+      "step": 780
+    },
+    {
+      "epoch": 2.338691588785047,
+      "grad_norm": 0.5596469044685364,
+      "learning_rate": 1.2633499480430568e-06,
+      "loss": 0.3319,
+      "step": 782
+    },
+    {
+      "epoch": 2.344672897196262,
+      "grad_norm": 0.6204531192779541,
+      "learning_rate": 1.2414817064892964e-06,
+      "loss": 0.3597,
+      "step": 784
+    },
+    {
+      "epoch": 2.3506542056074764,
+      "grad_norm": 0.5784049034118652,
+      "learning_rate": 1.2197775291227887e-06,
+      "loss": 0.3432,
+      "step": 786
+    },
+    {
+      "epoch": 2.3566355140186914,
+      "grad_norm": 0.603003203868866,
+      "learning_rate": 1.198238363358959e-06,
+      "loss": 0.3752,
+      "step": 788
+    },
+    {
+      "epoch": 2.3626168224299064,
+      "grad_norm": 0.5979328751564026,
+      "learning_rate": 1.1768651494102674e-06,
+      "loss": 0.3478,
+      "step": 790
+    },
+    {
+      "epoch": 2.3685981308411215,
+      "grad_norm": 0.594260036945343,
+      "learning_rate": 1.1556588202451613e-06,
+      "loss": 0.3485,
+      "step": 792
+    },
+    {
+      "epoch": 2.3745794392523365,
+      "grad_norm": 0.555630624294281,
+      "learning_rate": 1.1346203015473501e-06,
+      "loss": 0.3182,
+      "step": 794
+    },
+    {
+      "epoch": 2.3805607476635515,
+      "grad_norm": 0.6160412430763245,
+      "learning_rate": 1.1137505116754027e-06,
+      "loss": 0.3752,
+      "step": 796
+    },
+    {
+      "epoch": 2.3865420560747665,
+      "grad_norm": 0.5955886244773865,
+      "learning_rate": 1.0930503616226495e-06,
+      "loss": 0.3486,
+      "step": 798
+    },
+    {
+      "epoch": 2.392523364485981,
+      "grad_norm": 0.5942765474319458,
+      "learning_rate": 1.0725207549774313e-06,
+      "loss": 0.3507,
+      "step": 800
+    },
+    {
+      "epoch": 2.398504672897196,
+      "grad_norm": 0.6559773683547974,
+      "learning_rate": 1.0521625878836417e-06,
+      "loss": 0.3663,
+      "step": 802
+    },
+    {
+      "epoch": 2.404485981308411,
+      "grad_norm": 0.6017698645591736,
+      "learning_rate": 1.0319767490016196e-06,
+      "loss": 0.3359,
+      "step": 804
+    },
+    {
+      "epoch": 2.410467289719626,
+      "grad_norm": 0.6142606735229492,
+      "learning_rate": 1.0119641194693568e-06,
+      "loss": 0.3439,
+      "step": 806
+    },
+    {
+      "epoch": 2.4164485981308412,
+      "grad_norm": 0.6043434143066406,
+      "learning_rate": 9.921255728640244e-07,
+      "loss": 0.3726,
+      "step": 808
+    },
+    {
+      "epoch": 2.4224299065420563,
+      "grad_norm": 0.6003604531288147,
+      "learning_rate": 9.724619751638598e-07,
+      "loss": 0.3435,
+      "step": 810
+    },
+    {
+      "epoch": 2.428411214953271,
+      "grad_norm": 0.6148618459701538,
+      "learning_rate": 9.52974184710344e-07,
+      "loss": 0.3454,
+      "step": 812
+    },
+    {
+      "epoch": 2.434392523364486,
+      "grad_norm": 0.5786988139152527,
+      "learning_rate": 9.336630521707507e-07,
+      "loss": 0.3599,
+      "step": 814
+    },
+    {
+      "epoch": 2.440373831775701,
+      "grad_norm": 0.6036843657493591,
+      "learning_rate": 9.145294205010058e-07,
+      "loss": 0.3551,
+      "step": 816
+    },
+    {
+      "epoch": 2.446355140186916,
+      "grad_norm": 0.5846954584121704,
+      "learning_rate": 8.955741249088895e-07,
+      "loss": 0.3667,
+      "step": 818
+    },
+    {
+      "epoch": 2.452336448598131,
+      "grad_norm": 0.5913165807723999,
+      "learning_rate": 8.767979928175846e-07,
+      "loss": 0.3376,
+      "step": 820
+    },
+    {
+      "epoch": 2.458317757009346,
+      "grad_norm": 0.6217696666717529,
+      "learning_rate": 8.582018438295553e-07,
+      "loss": 0.3329,
+      "step": 822
+    },
+    {
+      "epoch": 2.464299065420561,
+      "grad_norm": 0.6013655066490173,
+      "learning_rate": 8.397864896907665e-07,
+      "loss": 0.3662,
+      "step": 824
+    },
+    {
+      "epoch": 2.4702803738317756,
+      "grad_norm": 0.6371009349822998,
+      "learning_rate": 8.21552734255261e-07,
+      "loss": 0.3659,
+      "step": 826
+    },
+    {
+      "epoch": 2.4762616822429906,
+      "grad_norm": 0.5932595729827881,
+      "learning_rate": 8.035013734500557e-07,
+      "loss": 0.3604,
+      "step": 828
+    },
+    {
+      "epoch": 2.4822429906542056,
+      "grad_norm": 0.5930650234222412,
+      "learning_rate": 7.856331952404117e-07,
+      "loss": 0.3512,
+      "step": 830
+    },
+    {
+      "epoch": 2.4882242990654206,
+      "grad_norm": 0.5936273336410522,
+      "learning_rate": 7.679489795954282e-07,
+      "loss": 0.3622,
+      "step": 832
+    },
+    {
+      "epoch": 2.4942056074766357,
+      "grad_norm": 0.5910542607307434,
+      "learning_rate": 7.504494984540033e-07,
+      "loss": 0.3709,
+      "step": 834
+    },
+    {
+      "epoch": 2.5001869158878502,
+      "grad_norm": 0.5870150923728943,
+      "learning_rate": 7.331355156911352e-07,
+      "loss": 0.3611,
+      "step": 836
+    },
+    {
+      "epoch": 2.5061682242990653,
+      "grad_norm": 0.6222584843635559,
+      "learning_rate": 7.16007787084575e-07,
+      "loss": 0.36,
+      "step": 838
+    },
+    {
+      "epoch": 2.5121495327102803,
+      "grad_norm": 0.6077357530593872,
+      "learning_rate": 6.990670602818412e-07,
+      "loss": 0.3705,
+      "step": 840
+    },
+    {
+      "epoch": 2.5181308411214953,
+      "grad_norm": 0.5924935340881348,
+      "learning_rate": 6.823140747675816e-07,
+      "loss": 0.3592,
+      "step": 842
+    },
+    {
+      "epoch": 2.5241121495327103,
+      "grad_norm": 0.5799605846405029,
+      "learning_rate": 6.657495618312926e-07,
+      "loss": 0.355,
+      "step": 844
+    },
+    {
+      "epoch": 2.5300934579439254,
+      "grad_norm": 0.5925043821334839,
+      "learning_rate": 6.493742445354012e-07,
+      "loss": 0.3441,
+      "step": 846
+    },
+    {
+      "epoch": 2.5360747663551404,
+      "grad_norm": 0.590898334980011,
+      "learning_rate": 6.331888376836948e-07,
+      "loss": 0.3561,
+      "step": 848
+    },
+    {
+      "epoch": 2.542056074766355,
+      "grad_norm": 0.612479031085968,
+      "learning_rate": 6.171940477901305e-07,
+      "loss": 0.367,
+      "step": 850
+    },
+    {
+      "epoch": 2.54803738317757,
+      "grad_norm": 0.5892294645309448,
+      "learning_rate": 6.013905730479824e-07,
+      "loss": 0.3622,
+      "step": 852
+    },
+    {
+      "epoch": 2.554018691588785,
+      "grad_norm": 0.5982328653335571,
+      "learning_rate": 5.857791032993754e-07,
+      "loss": 0.3583,
+      "step": 854
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 0.5935385227203369,
+      "learning_rate": 5.703603200051661e-07,
+      "loss": 0.3603,
+      "step": 856
+    },
+    {
+      "epoch": 2.565981308411215,
+      "grad_norm": 0.6001613736152649,
+      "learning_rate": 5.551348962151965e-07,
+      "loss": 0.3512,
+      "step": 858
+    },
+    {
+      "epoch": 2.5719626168224297,
+      "grad_norm": 0.5811797976493835,
+      "learning_rate": 5.401034965389202e-07,
+      "loss": 0.3588,
+      "step": 860
+    },
+    {
+      "epoch": 2.5779439252336447,
+      "grad_norm": 0.6029683947563171,
+      "learning_rate": 5.252667771163827e-07,
+      "loss": 0.3588,
+      "step": 862
+    },
+    {
+      "epoch": 2.5839252336448597,
+      "grad_norm": 0.5932782292366028,
+      "learning_rate": 5.106253855895865e-07,
+      "loss": 0.3701,
+      "step": 864
+    },
+    {
+      "epoch": 2.5899065420560747,
+      "grad_norm": 0.5977774262428284,
+      "learning_rate": 4.961799610742202e-07,
+      "loss": 0.3494,
+      "step": 866
+    },
+    {
+      "epoch": 2.5958878504672898,
+      "grad_norm": 0.6161970496177673,
+      "learning_rate": 4.819311341317546e-07,
+      "loss": 0.3631,
+      "step": 868
+    },
+    {
+      "epoch": 2.601869158878505,
+      "grad_norm": 0.5860589742660522,
+      "learning_rate": 4.678795267419267e-07,
+      "loss": 0.3621,
+      "step": 870
+    },
+    {
+      "epoch": 2.60785046728972,
+      "grad_norm": 0.6047132611274719,
+      "learning_rate": 4.540257522755809e-07,
+      "loss": 0.3437,
+      "step": 872
+    },
+    {
+      "epoch": 2.613831775700935,
+      "grad_norm": 0.60504549741745,
+      "learning_rate": 4.4037041546790115e-07,
+      "loss": 0.3459,
+      "step": 874
+    },
+    {
+      "epoch": 2.6198130841121494,
+      "grad_norm": 0.5911012887954712,
+      "learning_rate": 4.2691411239201007e-07,
+      "loss": 0.366,
+      "step": 876
+    },
+    {
+      "epoch": 2.6257943925233644,
+      "grad_norm": 0.6090118288993835,
+      "learning_rate": 4.1365743043294936e-07,
+      "loss": 0.3618,
+      "step": 878
+    },
+    {
+      "epoch": 2.6317757009345795,
+      "grad_norm": 0.5937606692314148,
+      "learning_rate": 4.006009482620432e-07,
+      "loss": 0.3594,
+      "step": 880
+    },
+    {
+      "epoch": 2.6377570093457945,
+      "grad_norm": 0.5993775725364685,
+      "learning_rate": 3.8774523581163236e-07,
+      "loss": 0.3503,
+      "step": 882
+    },
+    {
+      "epoch": 2.643738317757009,
+      "grad_norm": 0.6125180125236511,
+      "learning_rate": 3.750908542502013e-07,
+      "loss": 0.3646,
+      "step": 884
+    },
+    {
+      "epoch": 2.649719626168224,
+      "grad_norm": 0.6050174832344055,
+      "learning_rate": 3.6263835595788276e-07,
+      "loss": 0.371,
+      "step": 886
+    },
+    {
+      "epoch": 2.655700934579439,
+      "grad_norm": 2.164024829864502,
+      "learning_rate": 3.5038828450233874e-07,
+      "loss": 0.3525,
+      "step": 888
+    },
+    {
+      "epoch": 2.661682242990654,
+      "grad_norm": 0.6173498034477234,
+      "learning_rate": 3.3834117461504224e-07,
+      "loss": 0.3629,
+      "step": 890
+    },
+    {
+      "epoch": 2.667663551401869,
+      "grad_norm": 0.6122630834579468,
+      "learning_rate": 3.2649755216792866e-07,
+      "loss": 0.361,
+      "step": 892
+    },
+    {
+      "epoch": 2.673644859813084,
+      "grad_norm": 0.6037364602088928,
+      "learning_rate": 3.1485793415044483e-07,
+      "loss": 0.365,
+      "step": 894
+    },
+    {
+      "epoch": 2.679626168224299,
+      "grad_norm": 0.6137922406196594,
+      "learning_rate": 3.034228286469804e-07,
+      "loss": 0.3784,
+      "step": 896
+    },
+    {
+      "epoch": 2.6856074766355142,
+      "grad_norm": 0.5984251499176025,
+      "learning_rate": 2.9219273481468634e-07,
+      "loss": 0.3659,
+      "step": 898
+    },
+    {
+      "epoch": 2.691588785046729,
+      "grad_norm": 0.6204813122749329,
+      "learning_rate": 2.811681428616919e-07,
+      "loss": 0.3873,
+      "step": 900
+    },
+    {
+      "epoch": 2.697570093457944,
+      "grad_norm": 0.5879127383232117,
+      "learning_rate": 2.703495340257017e-07,
+      "loss": 0.3623,
+      "step": 902
+    },
+    {
+      "epoch": 2.703551401869159,
+      "grad_norm": 0.6163719296455383,
+      "learning_rate": 2.597373805529918e-07,
+      "loss": 0.3714,
+      "step": 904
+    },
+    {
+      "epoch": 2.709532710280374,
+      "grad_norm": 0.6320078372955322,
+      "learning_rate": 2.4933214567779473e-07,
+      "loss": 0.3716,
+      "step": 906
+    },
+    {
+      "epoch": 2.715514018691589,
+      "grad_norm": 0.6004301309585571,
+      "learning_rate": 2.39134283602076e-07,
+      "loss": 0.359,
+      "step": 908
+    },
+    {
+      "epoch": 2.7214953271028035,
+      "grad_norm": 0.6041020750999451,
+      "learning_rate": 2.2914423947571263e-07,
+      "loss": 0.3774,
+      "step": 910
+    },
+    {
+      "epoch": 2.7274766355140185,
+      "grad_norm": 0.6102595329284668,
+      "learning_rate": 2.193624493770591e-07,
+      "loss": 0.3776,
+      "step": 912
+    },
+    {
+      "epoch": 2.7334579439252336,
+      "grad_norm": 0.5953677296638489,
+      "learning_rate": 2.0978934029391052e-07,
+      "loss": 0.346,
+      "step": 914
+    },
+    {
+      "epoch": 2.7394392523364486,
+      "grad_norm": 0.6040077805519104,
+      "learning_rate": 2.0042533010486753e-07,
+      "loss": 0.3658,
+      "step": 916
+    },
+    {
+      "epoch": 2.7454205607476636,
+      "grad_norm": 0.5858832001686096,
+      "learning_rate": 1.9127082756109138e-07,
+      "loss": 0.34,
+      "step": 918
+    },
+    {
+      "epoch": 2.7514018691588786,
+      "grad_norm": 0.5845103859901428,
+      "learning_rate": 1.8232623226846546e-07,
+      "loss": 0.3475,
+      "step": 920
+    },
+    {
+      "epoch": 2.7573831775700937,
+      "grad_norm": 0.5982747673988342,
+      "learning_rate": 1.7359193467014812e-07,
+      "loss": 0.3724,
+      "step": 922
+    },
+    {
+      "epoch": 2.7633644859813087,
+      "grad_norm": 0.5818347930908203,
+      "learning_rate": 1.6506831602953298e-07,
+      "loss": 0.359,
+      "step": 924
+    },
+    {
+      "epoch": 2.7693457943925233,
+      "grad_norm": 0.628298819065094,
+      "learning_rate": 1.5675574841360343e-07,
+      "loss": 0.3561,
+      "step": 926
+    },
+    {
+      "epoch": 2.7753271028037383,
+      "grad_norm": 0.5835368037223816,
+      "learning_rate": 1.4865459467669262e-07,
+      "loss": 0.3727,
+      "step": 928
+    },
+    {
+      "epoch": 2.7813084112149533,
+      "grad_norm": 0.5914537906646729,
+      "learning_rate": 1.407652084446459e-07,
+      "loss": 0.3622,
+      "step": 930
+    },
+    {
+      "epoch": 2.7872897196261683,
+      "grad_norm": 0.6176457405090332,
+      "learning_rate": 1.330879340993796e-07,
+      "loss": 0.3387,
+      "step": 932
+    },
+    {
+      "epoch": 2.793271028037383,
+      "grad_norm": 0.5938091278076172,
+      "learning_rate": 1.2562310676385524e-07,
+      "loss": 0.3413,
+      "step": 934
+    },
+    {
+      "epoch": 2.799252336448598,
+      "grad_norm": 0.6224868893623352,
+      "learning_rate": 1.183710522874454e-07,
+      "loss": 0.3613,
+      "step": 936
+    },
+    {
+      "epoch": 2.805233644859813,
+      "grad_norm": 0.5785969495773315,
+      "learning_rate": 1.1133208723171251e-07,
+      "loss": 0.3766,
+      "step": 938
+    },
+    {
+      "epoch": 2.811214953271028,
+      "grad_norm": 0.5937323570251465,
+      "learning_rate": 1.0450651885659024e-07,
+      "loss": 0.3488,
+      "step": 940
+    },
+    {
+      "epoch": 2.817196261682243,
+      "grad_norm": 0.5992870330810547,
+      "learning_rate": 9.789464510697011e-08,
+      "loss": 0.3479,
+      "step": 942
+    },
+    {
+      "epoch": 2.823177570093458,
+      "grad_norm": 0.5977771878242493,
+      "learning_rate": 9.149675459969809e-08,
+      "loss": 0.3614,
+      "step": 944
+    },
+    {
+      "epoch": 2.829158878504673,
+      "grad_norm": 0.6392165422439575,
+      "learning_rate": 8.531312661097468e-08,
+      "loss": 0.3604,
+      "step": 946
+    },
+    {
+      "epoch": 2.835140186915888,
+      "grad_norm": 0.5865116119384766,
+      "learning_rate": 7.934403106416245e-08,
+      "loss": 0.3402,
+      "step": 948
+    },
+    {
+      "epoch": 2.8411214953271027,
+      "grad_norm": 0.5936951637268066,
+      "learning_rate": 7.358972851800705e-08,
+      "loss": 0.3661,
+      "step": 950
+    },
+    {
+      "epoch": 2.8471028037383177,
+      "grad_norm": 0.6197953224182129,
+      "learning_rate": 6.805047015526089e-08,
+      "loss": 0.3765,
+      "step": 952
+    },
+    {
+      "epoch": 2.8530841121495327,
+      "grad_norm": 0.6130885481834412,
+      "learning_rate": 6.272649777171902e-08,
+      "loss": 0.3635,
+      "step": 954
+    },
+    {
+      "epoch": 2.8590654205607478,
+      "grad_norm": 0.6389986872673035,
+      "learning_rate": 5.761804376566649e-08,
+      "loss": 0.3883,
+      "step": 956
+    },
+    {
+      "epoch": 2.8650467289719628,
+      "grad_norm": 0.6116965413093567,
+      "learning_rate": 5.272533112773093e-08,
+      "loss": 0.3438,
+      "step": 958
+    },
+    {
+      "epoch": 2.8710280373831774,
+      "grad_norm": 0.6007145643234253,
+      "learning_rate": 4.804857343114977e-08,
+      "loss": 0.3572,
+      "step": 960
+    },
+    {
+      "epoch": 2.8770093457943924,
+      "grad_norm": 0.6094311475753784,
+      "learning_rate": 4.358797482244936e-08,
+      "loss": 0.3671,
+      "step": 962
+    },
+    {
+      "epoch": 2.8829906542056074,
+      "grad_norm": 0.5995861291885376,
+      "learning_rate": 3.934373001253045e-08,
+      "loss": 0.3718,
+      "step": 964
+    },
+    {
+      "epoch": 2.8889719626168224,
+      "grad_norm": 0.5905115604400635,
+      "learning_rate": 3.5316024268172713e-08,
+      "loss": 0.3405,
+      "step": 966
+    },
+    {
+      "epoch": 2.8949532710280375,
+      "grad_norm": 0.6070297956466675,
+      "learning_rate": 3.15050334039424e-08,
+      "loss": 0.3656,
+      "step": 968
+    },
+    {
+      "epoch": 2.9009345794392525,
+      "grad_norm": 0.5917677879333496,
+      "learning_rate": 2.791092377452287e-08,
+      "loss": 0.3464,
+      "step": 970
+    },
+    {
+      "epoch": 2.9069158878504675,
+      "grad_norm": 0.60396808385849,
+      "learning_rate": 2.4533852267450976e-08,
+      "loss": 0.3566,
+      "step": 972
+    },
+    {
+      "epoch": 2.912897196261682,
+      "grad_norm": 0.6098156571388245,
+      "learning_rate": 2.1373966296266978e-08,
+      "loss": 0.3554,
+      "step": 974
+    },
+    {
+      "epoch": 2.918878504672897,
+      "grad_norm": 0.5947542190551758,
+      "learning_rate": 1.8431403794084148e-08,
+      "loss": 0.3857,
+      "step": 976
+    },
+    {
+      "epoch": 2.924859813084112,
+      "grad_norm": 0.6109650135040283,
+      "learning_rate": 1.5706293207561896e-08,
+      "loss": 0.3597,
+      "step": 978
+    },
+    {
+      "epoch": 2.930841121495327,
+      "grad_norm": 0.5936993360519409,
+      "learning_rate": 1.3198753491305283e-08,
+      "loss": 0.3639,
+      "step": 980
+    },
+    {
+      "epoch": 2.936822429906542,
+      "grad_norm": 0.6051398515701294,
+      "learning_rate": 1.0908894102666378e-08,
+      "loss": 0.3712,
+      "step": 982
+    },
+    {
+      "epoch": 2.9428037383177568,
+      "grad_norm": 0.598733127117157,
+      "learning_rate": 8.836814996971977e-09,
+      "loss": 0.3813,
+      "step": 984
+    },
+    {
+      "epoch": 2.948785046728972,
+      "grad_norm": 0.5998523831367493,
+      "learning_rate": 6.982606623155974e-09,
+      "loss": 0.3722,
+      "step": 986
+    },
+    {
+      "epoch": 2.954766355140187,
+      "grad_norm": 0.577263593673706,
+      "learning_rate": 5.346349919813088e-09,
+      "loss": 0.3459,
+      "step": 988
+    },
+    {
+      "epoch": 2.960747663551402,
+      "grad_norm": 0.7024447917938232,
+      "learning_rate": 3.928116311666119e-09,
+      "loss": 0.35,
+      "step": 990
+    },
+    {
+      "epoch": 2.966728971962617,
+      "grad_norm": 0.6336097121238708,
+      "learning_rate": 2.727967706447343e-09,
+      "loss": 0.3758,
+      "step": 992
+    },
+    {
+      "epoch": 2.972710280373832,
+      "grad_norm": 0.6015500426292419,
+      "learning_rate": 1.7459564921962213e-09,
+      "loss": 0.3569,
+      "step": 994
+    },
+    {
+      "epoch": 2.978691588785047,
+      "grad_norm": 0.6137949824333191,
+      "learning_rate": 9.821255349734548e-10,
+      "loss": 0.3711,
+      "step": 996
+    },
+    {
+      "epoch": 2.984672897196262,
+      "grad_norm": 0.5941304564476013,
+      "learning_rate": 4.3650817698803747e-10,
+      "loss": 0.3532,
+      "step": 998
+    },
+    {
+      "epoch": 2.9906542056074765,
+      "grad_norm": 0.6001025438308716,
+      "learning_rate": 1.0912823514397374e-10,
+      "loss": 0.3489,
+      "step": 1000
+    },
+    {
+      "epoch": 2.9966355140186915,
+      "grad_norm": 0.603685736656189,
+      "learning_rate": 0.0,
+      "loss": 0.3917,
+      "step": 1002
+    }
+  ],
+  "logging_steps": 2,
+  "max_steps": 1002,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1827834291784516e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2ec74c80b65fa6660137a6f5542a35ec448f6326b1cc4c9332453d17b01eeee
+size 7928

irish_deepseek_llama31_8b_translated_bespoke_stratos_17k/checkpoint-1002/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)