Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,98 @@
+---
+library_name: transformers
+license: other
+base_model: Qwen/Qwen2.5-7B-Instruct
+tags:
+- llama-factory
+- full
+- generated_from_trainer
+model-index:
+- name: Qwen2.5-7B-Instruct
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# Qwen2.5-7B-Instruct
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) on the self_ask_train_data dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.8082
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 16
+- total_eval_batch_size: 8
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 3.0
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 0.8711        | 0.0889 | 100  | 0.8548          |
+| 0.8052        | 0.1778 | 200  | 0.8143          |
+| 0.8071        | 0.2667 | 300  | 0.8015          |
+| 0.7912        | 0.3556 | 400  | 0.7962          |
+| 0.8086        | 0.4444 | 500  | 0.7914          |
+| 0.7614        | 0.5333 | 600  | 0.7859          |
+| 0.7758        | 0.6222 | 700  | 0.7828          |
+| 0.8026        | 0.7111 | 800  | 0.7794          |
+| 0.8           | 0.8    | 900  | 0.7757          |
+| 0.7568        | 0.8889 | 1000 | 0.7740          |
+| 0.7954        | 0.9778 | 1100 | 0.7712          |
+| 0.6518        | 1.0667 | 1200 | 0.7852          |
+| 0.6344        | 1.1556 | 1300 | 0.7862          |
+| 0.6181        | 1.2444 | 1400 | 0.7869          |
+| 0.6511        | 1.3333 | 1500 | 0.7798          |
+| 0.6341        | 1.4222 | 1600 | 0.7812          |
+| 0.6537        | 1.5111 | 1700 | 0.7794          |
+| 0.6626        | 1.6    | 1800 | 0.7780          |
+| 0.6116        | 1.6889 | 1900 | 0.7766          |
+| 0.6327        | 1.7778 | 2000 | 0.7731          |
+| 0.6168        | 1.8667 | 2100 | 0.7714          |
+| 0.6354        | 1.9556 | 2200 | 0.7699          |
+| 0.5238        | 2.0444 | 2300 | 0.8105          |
+| 0.4994        | 2.1333 | 2400 | 0.8090          |
+| 0.481         | 2.2222 | 2500 | 0.8098          |
+| 0.4976        | 2.3111 | 2600 | 0.8098          |
+| 0.5061        | 2.4    | 2700 | 0.8085          |
+| 0.5184        | 2.4889 | 2800 | 0.8096          |
+| 0.5024        | 2.5778 | 2900 | 0.8094          |
+| 0.5086        | 2.6667 | 3000 | 0.8081          |
+| 0.5008        | 2.7556 | 3100 | 0.8081          |
+| 0.5021        | 2.8444 | 3200 | 0.8082          |
+| 0.4808        | 2.9333 | 3300 | 0.8082          |
+
+
+### Framework versions
+
+- Transformers 4.46.1
+- Pytorch 2.5.1+cu124
+- Datasets 2.21.0
+- Tokenizers 0.20.3

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,12 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 0.8082394003868103,
+    "eval_runtime": 38.2561,
+    "eval_samples_per_second": 52.279,
+    "eval_steps_per_second": 6.535,
+    "total_flos": 173592713248768.0,
+    "train_loss": 0.6526138088791459,
+    "train_runtime": 5605.6539,
+    "train_samples_per_second": 9.633,
+    "train_steps_per_second": 0.602
+}

checkpoint-2000/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoint-2000/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.1",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

checkpoint-2000/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.46.1"
+}

checkpoint-2000/global_step2000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ec4963aefde519bfa290df416855fdc601c9985cc7340e1ac53cb70462f0f63
+size 11423430028

checkpoint-2000/global_step2000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:064bde90bac96d3ae0335a945eb1eb1fb6923966b8bdd7b0f98d6e9f533a9063
+size 11423430028

checkpoint-2000/global_step2000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ba6d0fa5d670bf71d22371e3ae39f76aeee315d30bf1f50f5f27a468968caea
+size 11423430028

checkpoint-2000/global_step2000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfb3a5d0aa42bf46113213b5ea8702920360d047cb22772bb2afdf172b623fae
+size 11423430028

checkpoint-2000/global_step2000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fe870c29b69fc1be568db01d80f42898cd0aa51cab50eb94b5736b7acf7f594
+size 11423430028

checkpoint-2000/global_step2000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b74506576e2c6904d2179eed1293f617bdf98f1a8a6e35c4040d03eaad28c1c
+size 11423430028

checkpoint-2000/global_step2000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9a15c6a39b6c79dd436862725da9802df29b9e8d11ef41b0c52c8a4d055454e
+size 11423430028

checkpoint-2000/global_step2000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d41acc996c67d4b1668077f12ae9f29d5b0cc7cead1ed3dd45e144dc2cbe7694
+size 11423430028

checkpoint-2000/global_step2000/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24b959dba227c4ddf656d91cd1c38459188fe407c0509cc9a4cb8727585f9603
+size 168277

checkpoint-2000/global_step2000/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f56f48d4df025ede26974d4018e3e7d036f2c68f0a089fdb6bc322e93c1ac436
+size 168277

checkpoint-2000/global_step2000/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75d30916d5f48e6cbe0daf8b4323cb578b19123f8152a16b71502fe062eec64b
+size 168277

checkpoint-2000/global_step2000/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:641b0d745bffa34b6dedda7bbb1812e06d2207af837e1c4d821dd18a24c4f607
+size 168277

checkpoint-2000/global_step2000/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8979044e5d7e6e9da13496613a171684c603fb1633c7d65ba939960e9cad5775
+size 168277

checkpoint-2000/global_step2000/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18e06f547a53cb0b67d9fd2cbaaf1f4e8d3065a3d0fb22684856e2fe26bd2ef7
+size 168277

checkpoint-2000/global_step2000/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b00790723f4f010faf5efae4eca7d12a5ab043416eb039b5364193bdd43f0e70
+size 168277

checkpoint-2000/global_step2000/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:434e832401e21f7e825d2656338a6ed1935713151b690304c82065435d60aa8a
+size 168277

checkpoint-2000/latest

@@ -0,0 +1 @@
+global_step2000

checkpoint-2000/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce61da88da5accc0db8f38ba80f8c1eccaeaed122cc846cd4689d5a9130071f5
+size 4877660776

checkpoint-2000/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69f630b9731bc146d762a5f9046afa214eddbcbbd2996fca1827fe94968e2416
+size 4932751008

checkpoint-2000/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3967a7a0bb19277e6d0d686b61337c0287eeaa27189a4fddc56ba1a24814f378
+size 4330865200

checkpoint-2000/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b2f1d559070bb819a86bde59ed06d6746d521439dd06c165e7c333e7042cd72
+size 1089994880

checkpoint-2000/model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 15231233024
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors"
+  }
+}

checkpoint-2000/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ffbd38f2626212a6ee67166039a4e916fc424a3ef78a112b81ecba34404f3d1
+size 15984

checkpoint-2000/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:761cc4f38afe955241d2348fca72dd9656d9ebe889e2a62241d58bd4bc402ab3
+size 15984

checkpoint-2000/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df49e54f3e60ca7b97616f5b3ec776cb88dd34df15356bfe0565e073195eabb5
+size 15984

checkpoint-2000/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a34a02b568d14f0c9040613f420faf5a53edb4c287e30c4c7c5812d2c661932
+size 15984

checkpoint-2000/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbb893a42ae156daa2dddcfcf2ef773c576ec3c274aaf9e5a216bd1d659a190b
+size 15984

checkpoint-2000/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b966d4dca42c479a7d82dd6ce6a32209ee7e19048adffff6ca6d5cbfafe89c52
+size 15984

checkpoint-2000/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfdec22eeccd05416d2a00acd09a775df394351b2aa4e3b1d32df8f7f4a8b07f
+size 15984

checkpoint-2000/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89e852c764f530a6e44842564809b78387dfa34f7ec6fcece34742866c5d452e
+size 15984

checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc6c2bcd9b76d33bd8c89a00cca67d819437a66a3d71f03aee9d963b3a1b6c2c
+size 1064

checkpoint-2000/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-2000/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

checkpoint-2000/tokenizer_config.json

@@ -0,0 +1,208 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-2000/trainer_state.json

@@ -0,0 +1,1593 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.7777777777777777,
+  "eval_steps": 100,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.008888888888888889,
+      "grad_norm": 6.312672588322296,
+      "learning_rate": 1.4792899408284025e-07,
+      "loss": 1.1997,
+      "step": 10
+    },
+    {
+      "epoch": 0.017777777777777778,
+      "grad_norm": 6.5395437094474875,
+      "learning_rate": 2.958579881656805e-07,
+      "loss": 1.2188,
+      "step": 20
+    },
+    {
+      "epoch": 0.02666666666666667,
+      "grad_norm": 5.051656620162693,
+      "learning_rate": 4.4378698224852073e-07,
+      "loss": 1.1654,
+      "step": 30
+    },
+    {
+      "epoch": 0.035555555555555556,
+      "grad_norm": 3.245773084025693,
+      "learning_rate": 5.91715976331361e-07,
+      "loss": 1.0942,
+      "step": 40
+    },
+    {
+      "epoch": 0.044444444444444446,
+      "grad_norm": 2.594184571301331,
+      "learning_rate": 7.396449704142013e-07,
+      "loss": 1.0498,
+      "step": 50
+    },
+    {
+      "epoch": 0.05333333333333334,
+      "grad_norm": 2.168963351286361,
+      "learning_rate": 8.875739644970415e-07,
+      "loss": 0.9403,
+      "step": 60
+    },
+    {
+      "epoch": 0.06222222222222222,
+      "grad_norm": 1.9004139854270057,
+      "learning_rate": 1.0355029585798817e-06,
+      "loss": 0.9099,
+      "step": 70
+    },
+    {
+      "epoch": 0.07111111111111111,
+      "grad_norm": 2.2242455117026494,
+      "learning_rate": 1.183431952662722e-06,
+      "loss": 0.8998,
+      "step": 80
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 1.8519949378838332,
+      "learning_rate": 1.3313609467455623e-06,
+      "loss": 0.859,
+      "step": 90
+    },
+    {
+      "epoch": 0.08888888888888889,
+      "grad_norm": 1.8728553263645957,
+      "learning_rate": 1.4792899408284026e-06,
+      "loss": 0.8711,
+      "step": 100
+    },
+    {
+      "epoch": 0.08888888888888889,
+      "eval_loss": 0.8548257946968079,
+      "eval_runtime": 38.267,
+      "eval_samples_per_second": 52.264,
+      "eval_steps_per_second": 6.533,
+      "step": 100
+    },
+    {
+      "epoch": 0.09777777777777778,
+      "grad_norm": 2.116764307544036,
+      "learning_rate": 1.6272189349112426e-06,
+      "loss": 0.8557,
+      "step": 110
+    },
+    {
+      "epoch": 0.10666666666666667,
+      "grad_norm": 2.0364313595247467,
+      "learning_rate": 1.775147928994083e-06,
+      "loss": 0.8619,
+      "step": 120
+    },
+    {
+      "epoch": 0.11555555555555555,
+      "grad_norm": 1.7761221870002313,
+      "learning_rate": 1.9230769230769234e-06,
+      "loss": 0.86,
+      "step": 130
+    },
+    {
+      "epoch": 0.12444444444444444,
+      "grad_norm": 1.8902384905200484,
+      "learning_rate": 2.0710059171597635e-06,
+      "loss": 0.8483,
+      "step": 140
+    },
+    {
+      "epoch": 0.13333333333333333,
+      "grad_norm": 2.0970800465534793,
+      "learning_rate": 2.2189349112426035e-06,
+      "loss": 0.8841,
+      "step": 150
+    },
+    {
+      "epoch": 0.14222222222222222,
+      "grad_norm": 2.0388812214422005,
+      "learning_rate": 2.366863905325444e-06,
+      "loss": 0.8491,
+      "step": 160
+    },
+    {
+      "epoch": 0.1511111111111111,
+      "grad_norm": 1.9949400049388208,
+      "learning_rate": 2.5147928994082845e-06,
+      "loss": 0.8252,
+      "step": 170
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.7986729179125904,
+      "learning_rate": 2.6627218934911246e-06,
+      "loss": 0.8345,
+      "step": 180
+    },
+    {
+      "epoch": 0.1688888888888889,
+      "grad_norm": 1.8870696385512356,
+      "learning_rate": 2.8106508875739646e-06,
+      "loss": 0.8259,
+      "step": 190
+    },
+    {
+      "epoch": 0.17777777777777778,
+      "grad_norm": 1.8573694639357368,
+      "learning_rate": 2.958579881656805e-06,
+      "loss": 0.8052,
+      "step": 200
+    },
+    {
+      "epoch": 0.17777777777777778,
+      "eval_loss": 0.8143225312232971,
+      "eval_runtime": 37.9982,
+      "eval_samples_per_second": 52.634,
+      "eval_steps_per_second": 6.579,
+      "step": 200
+    },
+    {
+      "epoch": 0.18666666666666668,
+      "grad_norm": 1.8655947177959404,
+      "learning_rate": 3.106508875739645e-06,
+      "loss": 0.827,
+      "step": 210
+    },
+    {
+      "epoch": 0.19555555555555557,
+      "grad_norm": 1.870936582770945,
+      "learning_rate": 3.2544378698224853e-06,
+      "loss": 0.813,
+      "step": 220
+    },
+    {
+      "epoch": 0.20444444444444446,
+      "grad_norm": 2.109320545375679,
+      "learning_rate": 3.4023668639053257e-06,
+      "loss": 0.8174,
+      "step": 230
+    },
+    {
+      "epoch": 0.21333333333333335,
+      "grad_norm": 1.8097321542077869,
+      "learning_rate": 3.550295857988166e-06,
+      "loss": 0.8182,
+      "step": 240
+    },
+    {
+      "epoch": 0.2222222222222222,
+      "grad_norm": 1.728199611021706,
+      "learning_rate": 3.6982248520710063e-06,
+      "loss": 0.8123,
+      "step": 250
+    },
+    {
+      "epoch": 0.2311111111111111,
+      "grad_norm": 2.0315555925984348,
+      "learning_rate": 3.846153846153847e-06,
+      "loss": 0.7728,
+      "step": 260
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 1.5850711075399915,
+      "learning_rate": 3.9940828402366864e-06,
+      "loss": 0.7917,
+      "step": 270
+    },
+    {
+      "epoch": 0.24888888888888888,
+      "grad_norm": 1.927987824759959,
+      "learning_rate": 4.142011834319527e-06,
+      "loss": 0.8142,
+      "step": 280
+    },
+    {
+      "epoch": 0.2577777777777778,
+      "grad_norm": 1.9163193176432178,
+      "learning_rate": 4.289940828402367e-06,
+      "loss": 0.8177,
+      "step": 290
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "grad_norm": 1.9493054796009528,
+      "learning_rate": 4.437869822485207e-06,
+      "loss": 0.8071,
+      "step": 300
+    },
+    {
+      "epoch": 0.26666666666666666,
+      "eval_loss": 0.8015236854553223,
+      "eval_runtime": 38.0122,
+      "eval_samples_per_second": 52.615,
+      "eval_steps_per_second": 6.577,
+      "step": 300
+    },
+    {
+      "epoch": 0.27555555555555555,
+      "grad_norm": 1.7023378433784926,
+      "learning_rate": 4.5857988165680475e-06,
+      "loss": 0.8152,
+      "step": 310
+    },
+    {
+      "epoch": 0.28444444444444444,
+      "grad_norm": 1.8979776132841102,
+      "learning_rate": 4.733727810650888e-06,
+      "loss": 0.8301,
+      "step": 320
+    },
+    {
+      "epoch": 0.29333333333333333,
+      "grad_norm": 1.975097152658989,
+      "learning_rate": 4.8816568047337285e-06,
+      "loss": 0.8332,
+      "step": 330
+    },
+    {
+      "epoch": 0.3022222222222222,
+      "grad_norm": 1.8218939536712146,
+      "learning_rate": 4.99999464967688e-06,
+      "loss": 0.8286,
+      "step": 340
+    },
+    {
+      "epoch": 0.3111111111111111,
+      "grad_norm": 1.8977171195446734,
+      "learning_rate": 4.999807390772256e-06,
+      "loss": 0.8209,
+      "step": 350
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 1.880983857135489,
+      "learning_rate": 4.999352638611963e-06,
+      "loss": 0.8311,
+      "step": 360
+    },
+    {
+      "epoch": 0.3288888888888889,
+      "grad_norm": 2.085715913698874,
+      "learning_rate": 4.998630441857007e-06,
+      "loss": 0.8194,
+      "step": 370
+    },
+    {
+      "epoch": 0.3377777777777778,
+      "grad_norm": 1.8193588799474414,
+      "learning_rate": 4.997640877786446e-06,
+      "loss": 0.7961,
+      "step": 380
+    },
+    {
+      "epoch": 0.3466666666666667,
+      "grad_norm": 1.7395847960563042,
+      "learning_rate": 4.996384052289124e-06,
+      "loss": 0.8235,
+      "step": 390
+    },
+    {
+      "epoch": 0.35555555555555557,
+      "grad_norm": 1.7517335960292617,
+      "learning_rate": 4.994860099852339e-06,
+      "loss": 0.7912,
+      "step": 400
+    },
+    {
+      "epoch": 0.35555555555555557,
+      "eval_loss": 0.7962385416030884,
+      "eval_runtime": 38.0506,
+      "eval_samples_per_second": 52.562,
+      "eval_steps_per_second": 6.57,
+      "step": 400
+    },
+    {
+      "epoch": 0.36444444444444446,
+      "grad_norm": 1.7751508270993985,
+      "learning_rate": 4.993069183547456e-06,
+      "loss": 0.7956,
+      "step": 410
+    },
+    {
+      "epoch": 0.37333333333333335,
+      "grad_norm": 1.905940982272453,
+      "learning_rate": 4.991011495012451e-06,
+      "loss": 0.8383,
+      "step": 420
+    },
+    {
+      "epoch": 0.38222222222222224,
+      "grad_norm": 1.9604410876662892,
+      "learning_rate": 4.98868725443141e-06,
+      "loss": 0.8427,
+      "step": 430
+    },
+    {
+      "epoch": 0.39111111111111113,
+      "grad_norm": 1.8801968456938927,
+      "learning_rate": 4.986096710510968e-06,
+      "loss": 0.8253,
+      "step": 440
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 1.834581689663337,
+      "learning_rate": 4.9832401404536915e-06,
+      "loss": 0.8211,
+      "step": 450
+    },
+    {
+      "epoch": 0.4088888888888889,
+      "grad_norm": 1.816476724466639,
+      "learning_rate": 4.980117849928419e-06,
+      "loss": 0.792,
+      "step": 460
+    },
+    {
+      "epoch": 0.4177777777777778,
+      "grad_norm": 1.7561196396819305,
+      "learning_rate": 4.976730173037556e-06,
+      "loss": 0.8199,
+      "step": 470
+    },
+    {
+      "epoch": 0.4266666666666667,
+      "grad_norm": 1.883523346070099,
+      "learning_rate": 4.973077472281319e-06,
+      "loss": 0.8085,
+      "step": 480
+    },
+    {
+      "epoch": 0.43555555555555553,
+      "grad_norm": 1.8002160041857591,
+      "learning_rate": 4.969160138518946e-06,
+      "loss": 0.7981,
+      "step": 490
+    },
+    {
+      "epoch": 0.4444444444444444,
+      "grad_norm": 1.8807969403322338,
+      "learning_rate": 4.964978590926879e-06,
+      "loss": 0.8086,
+      "step": 500
+    },
+    {
+      "epoch": 0.4444444444444444,
+      "eval_loss": 0.7913944721221924,
+      "eval_runtime": 37.9142,
+      "eval_samples_per_second": 52.751,
+      "eval_steps_per_second": 6.594,
+      "step": 500
+    },
+    {
+      "epoch": 0.4533333333333333,
+      "grad_norm": 1.7162663123083326,
+      "learning_rate": 4.960533276953902e-06,
+      "loss": 0.8121,
+      "step": 510
+    },
+    {
+      "epoch": 0.4622222222222222,
+      "grad_norm": 1.8966249612321189,
+      "learning_rate": 4.955824672273265e-06,
+      "loss": 0.7929,
+      "step": 520
+    },
+    {
+      "epoch": 0.4711111111111111,
+      "grad_norm": 1.9877589179045125,
+      "learning_rate": 4.950853280731785e-06,
+      "loss": 0.8092,
+      "step": 530
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 1.6886441919837933,
+      "learning_rate": 4.945619634295929e-06,
+      "loss": 0.8393,
+      "step": 540
+    },
+    {
+      "epoch": 0.4888888888888889,
+      "grad_norm": 1.6732855356580696,
+      "learning_rate": 4.940124292994895e-06,
+      "loss": 0.8036,
+      "step": 550
+    },
+    {
+      "epoch": 0.49777777777777776,
+      "grad_norm": 1.8728351825828837,
+      "learning_rate": 4.9343678448606816e-06,
+      "loss": 0.8118,
+      "step": 560
+    },
+    {
+      "epoch": 0.5066666666666667,
+      "grad_norm": 1.7910848529169285,
+      "learning_rate": 4.928350905865165e-06,
+      "loss": 0.7975,
+      "step": 570
+    },
+    {
+      "epoch": 0.5155555555555555,
+      "grad_norm": 1.8879257679795278,
+      "learning_rate": 4.92207411985419e-06,
+      "loss": 0.8133,
+      "step": 580
+    },
+    {
+      "epoch": 0.5244444444444445,
+      "grad_norm": 1.82915454709858,
+      "learning_rate": 4.915538158478674e-06,
+      "loss": 0.8131,
+      "step": 590
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 1.9822261774763363,
+      "learning_rate": 4.908743721122734e-06,
+      "loss": 0.7614,
+      "step": 600
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "eval_loss": 0.7859019637107849,
+      "eval_runtime": 37.9245,
+      "eval_samples_per_second": 52.736,
+      "eval_steps_per_second": 6.592,
+      "step": 600
+    },
+    {
+      "epoch": 0.5422222222222223,
+      "grad_norm": 1.7590845282435732,
+      "learning_rate": 4.901691534828853e-06,
+      "loss": 0.785,
+      "step": 610
+    },
+    {
+      "epoch": 0.5511111111111111,
+      "grad_norm": 1.7642623616476127,
+      "learning_rate": 4.894382354220077e-06,
+      "loss": 0.8031,
+      "step": 620
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 1.9487864823051528,
+      "learning_rate": 4.886816961419272e-06,
+      "loss": 0.7899,
+      "step": 630
+    },
+    {
+      "epoch": 0.5688888888888889,
+      "grad_norm": 1.8294787755068975,
+      "learning_rate": 4.8789961659654276e-06,
+      "loss": 0.8079,
+      "step": 640
+    },
+    {
+      "epoch": 0.5777777777777777,
+      "grad_norm": 1.8448471765270655,
+      "learning_rate": 4.870920804727034e-06,
+      "loss": 0.8053,
+      "step": 650
+    },
+    {
+      "epoch": 0.5866666666666667,
+      "grad_norm": 1.9086870869024488,
+      "learning_rate": 4.862591741812533e-06,
+      "loss": 0.7919,
+      "step": 660
+    },
+    {
+      "epoch": 0.5955555555555555,
+      "grad_norm": 1.8399676032417565,
+      "learning_rate": 4.8540098684778505e-06,
+      "loss": 0.8049,
+      "step": 670
+    },
+    {
+      "epoch": 0.6044444444444445,
+      "grad_norm": 1.7627163023842312,
+      "learning_rate": 4.845176103031035e-06,
+      "loss": 0.7853,
+      "step": 680
+    },
+    {
+      "epoch": 0.6133333333333333,
+      "grad_norm": 1.880754220932831,
+      "learning_rate": 4.836091390733983e-06,
+      "loss": 0.7671,
+      "step": 690
+    },
+    {
+      "epoch": 0.6222222222222222,
+      "grad_norm": 1.9327151521828667,
+      "learning_rate": 4.826756703701298e-06,
+      "loss": 0.7758,
+      "step": 700
+    },
+    {
+      "epoch": 0.6222222222222222,
+      "eval_loss": 0.7827751636505127,
+      "eval_runtime": 37.9583,
+      "eval_samples_per_second": 52.689,
+      "eval_steps_per_second": 6.586,
+      "step": 700
+    },
+    {
+      "epoch": 0.6311111111111111,
+      "grad_norm": 1.7085337797540057,
+      "learning_rate": 4.817173040796263e-06,
+      "loss": 0.7581,
+      "step": 710
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 1.8693393810766439,
+      "learning_rate": 4.807341427523969e-06,
+      "loss": 0.8092,
+      "step": 720
+    },
+    {
+      "epoch": 0.6488888888888888,
+      "grad_norm": 1.9224847774084641,
+      "learning_rate": 4.797262915921561e-06,
+      "loss": 0.7905,
+      "step": 730
+    },
+    {
+      "epoch": 0.6577777777777778,
+      "grad_norm": 1.7921622260237242,
+      "learning_rate": 4.7869385844456825e-06,
+      "loss": 0.8371,
+      "step": 740
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 1.7662786771403587,
+      "learning_rate": 4.776369537857062e-06,
+      "loss": 0.8039,
+      "step": 750
+    },
+    {
+      "epoch": 0.6755555555555556,
+      "grad_norm": 1.67555577960425,
+      "learning_rate": 4.765556907102306e-06,
+      "loss": 0.8153,
+      "step": 760
+    },
+    {
+      "epoch": 0.6844444444444444,
+      "grad_norm": 1.8971436028748803,
+      "learning_rate": 4.7545018491928755e-06,
+      "loss": 0.777,
+      "step": 770
+    },
+    {
+      "epoch": 0.6933333333333334,
+      "grad_norm": 1.6522195956975418,
+      "learning_rate": 4.743205547081281e-06,
+      "loss": 0.7826,
+      "step": 780
+    },
+    {
+      "epoch": 0.7022222222222222,
+      "grad_norm": 1.678020666686354,
+      "learning_rate": 4.731669209534504e-06,
+      "loss": 0.797,
+      "step": 790
+    },
+    {
+      "epoch": 0.7111111111111111,
+      "grad_norm": 1.8314853043295487,
+      "learning_rate": 4.719894071004645e-06,
+      "loss": 0.8026,
+      "step": 800
+    },
+    {
+      "epoch": 0.7111111111111111,
+      "eval_loss": 0.77940833568573,
+      "eval_runtime": 37.9303,
+      "eval_samples_per_second": 52.728,
+      "eval_steps_per_second": 6.591,
+      "step": 800
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 1.6824200358042496,
+      "learning_rate": 4.707881391496837e-06,
+      "loss": 0.7976,
+      "step": 810
+    },
+    {
+      "epoch": 0.7288888888888889,
+      "grad_norm": 1.7994312639466354,
+      "learning_rate": 4.695632456434414e-06,
+      "loss": 0.7883,
+      "step": 820
+    },
+    {
+      "epoch": 0.7377777777777778,
+      "grad_norm": 1.873881222655515,
+      "learning_rate": 4.683148576521363e-06,
+      "loss": 0.7901,
+      "step": 830
+    },
+    {
+      "epoch": 0.7466666666666667,
+      "grad_norm": 1.7081859666296215,
+      "learning_rate": 4.670431087602079e-06,
+      "loss": 0.7747,
+      "step": 840
+    },
+    {
+      "epoch": 0.7555555555555555,
+      "grad_norm": 1.817576578011179,
+      "learning_rate": 4.657481350518409e-06,
+      "loss": 0.8109,
+      "step": 850
+    },
+    {
+      "epoch": 0.7644444444444445,
+      "grad_norm": 1.827616370549138,
+      "learning_rate": 4.644300750964045e-06,
+      "loss": 0.8191,
+      "step": 860
+    },
+    {
+      "epoch": 0.7733333333333333,
+      "grad_norm": 1.783716604929134,
+      "learning_rate": 4.630890699336244e-06,
+      "loss": 0.7728,
+      "step": 870
+    },
+    {
+      "epoch": 0.7822222222222223,
+      "grad_norm": 1.8315774625359147,
+      "learning_rate": 4.6172526305849094e-06,
+      "loss": 0.7899,
+      "step": 880
+    },
+    {
+      "epoch": 0.7911111111111111,
+      "grad_norm": 1.9174012768682513,
+      "learning_rate": 4.603388004059037e-06,
+      "loss": 0.8334,
+      "step": 890
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.7128496253638654,
+      "learning_rate": 4.589298303350565e-06,
+      "loss": 0.8,
+      "step": 900
+    },
+    {
+      "epoch": 0.8,
+      "eval_loss": 0.7756551504135132,
+      "eval_runtime": 38.0779,
+      "eval_samples_per_second": 52.524,
+      "eval_steps_per_second": 6.565,
+      "step": 900
+    },
+    {
+      "epoch": 0.8088888888888889,
+      "grad_norm": 2.087290188435175,
+      "learning_rate": 4.574985036135613e-06,
+      "loss": 0.7719,
+      "step": 910
+    },
+    {
+      "epoch": 0.8177777777777778,
+      "grad_norm": 1.7974331962068042,
+      "learning_rate": 4.5604497340131635e-06,
+      "loss": 0.765,
+      "step": 920
+    },
+    {
+      "epoch": 0.8266666666666667,
+      "grad_norm": 1.722447066657561,
+      "learning_rate": 4.545693952341159e-06,
+      "loss": 0.8213,
+      "step": 930
+    },
+    {
+      "epoch": 0.8355555555555556,
+      "grad_norm": 1.8137932027496781,
+      "learning_rate": 4.5307192700700804e-06,
+      "loss": 0.8189,
+      "step": 940
+    },
+    {
+      "epoch": 0.8444444444444444,
+      "grad_norm": 1.7200813454468675,
+      "learning_rate": 4.515527289573986e-06,
+      "loss": 0.7995,
+      "step": 950
+    },
+    {
+      "epoch": 0.8533333333333334,
+      "grad_norm": 1.7492699579153432,
+      "learning_rate": 4.50011963647905e-06,
+      "loss": 0.8045,
+      "step": 960
+    },
+    {
+      "epoch": 0.8622222222222222,
+      "grad_norm": 1.8904981608159812,
+      "learning_rate": 4.484497959489608e-06,
+      "loss": 0.8056,
+      "step": 970
+    },
+    {
+      "epoch": 0.8711111111111111,
+      "grad_norm": 1.7842392668053924,
+      "learning_rate": 4.468663930211743e-06,
+      "loss": 0.7881,
+      "step": 980
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 1.8109751414374233,
+      "learning_rate": 4.452619242974408e-06,
+      "loss": 0.7913,
+      "step": 990
+    },
+    {
+      "epoch": 0.8888888888888888,
+      "grad_norm": 1.6941534156764066,
+      "learning_rate": 4.436365614648128e-06,
+      "loss": 0.7568,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8888888888888888,
+      "eval_loss": 0.7739697098731995,
+      "eval_runtime": 38.0044,
+      "eval_samples_per_second": 52.625,
+      "eval_steps_per_second": 6.578,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8977777777777778,
+      "grad_norm": 1.7398957715176837,
+      "learning_rate": 4.4199047844612825e-06,
+      "loss": 0.812,
+      "step": 1010
+    },
+    {
+      "epoch": 0.9066666666666666,
+      "grad_norm": 1.804683657893465,
+      "learning_rate": 4.4032385138139985e-06,
+      "loss": 0.8158,
+      "step": 1020
+    },
+    {
+      "epoch": 0.9155555555555556,
+      "grad_norm": 1.8261068083326866,
+      "learning_rate": 4.386368586089674e-06,
+      "loss": 0.7766,
+      "step": 1030
+    },
+    {
+      "epoch": 0.9244444444444444,
+      "grad_norm": 1.929492696716588,
+      "learning_rate": 4.369296806464141e-06,
+      "loss": 0.801,
+      "step": 1040
+    },
+    {
+      "epoch": 0.9333333333333333,
+      "grad_norm": 1.8472605566952744,
+      "learning_rate": 4.3520250017125076e-06,
+      "loss": 0.7876,
+      "step": 1050
+    },
+    {
+      "epoch": 0.9422222222222222,
+      "grad_norm": 1.790367907695698,
+      "learning_rate": 4.334555020013675e-06,
+      "loss": 0.7681,
+      "step": 1060
+    },
+    {
+      "epoch": 0.9511111111111111,
+      "grad_norm": 1.6599756385687388,
+      "learning_rate": 4.316888730752583e-06,
+      "loss": 0.8135,
+      "step": 1070
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 1.6291519979649327,
+      "learning_rate": 4.299028024320166e-06,
+      "loss": 0.7741,
+      "step": 1080
+    },
+    {
+      "epoch": 0.9688888888888889,
+      "grad_norm": 1.5442340466379858,
+      "learning_rate": 4.280974811911071e-06,
+      "loss": 0.8041,
+      "step": 1090
+    },
+    {
+      "epoch": 0.9777777777777777,
+      "grad_norm": 1.7271244824498846,
+      "learning_rate": 4.262731025319159e-06,
+      "loss": 0.7954,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9777777777777777,
+      "eval_loss": 0.771242082118988,
+      "eval_runtime": 38.2542,
+      "eval_samples_per_second": 52.282,
+      "eval_steps_per_second": 6.535,
+      "step": 1100
+    },
+    {
+      "epoch": 0.9866666666666667,
+      "grad_norm": 1.730188161467645,
+      "learning_rate": 4.244298616730781e-06,
+      "loss": 0.8021,
+      "step": 1110
+    },
+    {
+      "epoch": 0.9955555555555555,
+      "grad_norm": 1.739908218239288,
+      "learning_rate": 4.2256795585158894e-06,
+      "loss": 0.8171,
+      "step": 1120
+    },
+    {
+      "epoch": 1.0044444444444445,
+      "grad_norm": 1.6139566050826326,
+      "learning_rate": 4.2068758430169805e-06,
+      "loss": 0.708,
+      "step": 1130
+    },
+    {
+      "epoch": 1.0133333333333334,
+      "grad_norm": 1.626198851176862,
+      "learning_rate": 4.187889482335905e-06,
+      "loss": 0.6309,
+      "step": 1140
+    },
+    {
+      "epoch": 1.0222222222222221,
+      "grad_norm": 1.7768887610773263,
+      "learning_rate": 4.168722508118562e-06,
+      "loss": 0.6455,
+      "step": 1150
+    },
+    {
+      "epoch": 1.031111111111111,
+      "grad_norm": 1.5692860084569409,
+      "learning_rate": 4.1493769713374995e-06,
+      "loss": 0.6528,
+      "step": 1160
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 1.6046426624256729,
+      "learning_rate": 4.12985494207245e-06,
+      "loss": 0.6443,
+      "step": 1170
+    },
+    {
+      "epoch": 1.048888888888889,
+      "grad_norm": 1.741907151593938,
+      "learning_rate": 4.110158509288822e-06,
+      "loss": 0.6194,
+      "step": 1180
+    },
+    {
+      "epoch": 1.0577777777777777,
+      "grad_norm": 1.7983079569047793,
+      "learning_rate": 4.090289780614167e-06,
+      "loss": 0.6544,
+      "step": 1190
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 1.955188619876201,
+      "learning_rate": 4.070250882112652e-06,
+      "loss": 0.6518,
+      "step": 1200
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "eval_loss": 0.7851578593254089,
+      "eval_runtime": 38.1241,
+      "eval_samples_per_second": 52.46,
+      "eval_steps_per_second": 6.558,
+      "step": 1200
+    },
+    {
+      "epoch": 1.0755555555555556,
+      "grad_norm": 1.7232020856742132,
+      "learning_rate": 4.050043958057561e-06,
+      "loss": 0.6258,
+      "step": 1210
+    },
+    {
+      "epoch": 1.0844444444444445,
+      "grad_norm": 1.593862025253077,
+      "learning_rate": 4.029671170701841e-06,
+      "loss": 0.6347,
+      "step": 1220
+    },
+    {
+      "epoch": 1.0933333333333333,
+      "grad_norm": 1.8219516480738216,
+      "learning_rate": 4.009134700046735e-06,
+      "loss": 0.6266,
+      "step": 1230
+    },
+    {
+      "epoch": 1.1022222222222222,
+      "grad_norm": 1.714593831219954,
+      "learning_rate": 3.988436743608506e-06,
+      "loss": 0.6077,
+      "step": 1240
+    },
+    {
+      "epoch": 1.1111111111111112,
+      "grad_norm": 1.7671062357329015,
+      "learning_rate": 3.967579516183292e-06,
+      "loss": 0.6797,
+      "step": 1250
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 1.6809718048297995,
+      "learning_rate": 3.946565249610108e-06,
+      "loss": 0.6416,
+      "step": 1260
+    },
+    {
+      "epoch": 1.1288888888888888,
+      "grad_norm": 1.7070363329071652,
+      "learning_rate": 3.925396192532032e-06,
+      "loss": 0.6252,
+      "step": 1270
+    },
+    {
+      "epoch": 1.1377777777777778,
+      "grad_norm": 1.7723751757721016,
+      "learning_rate": 3.90407461015558e-06,
+      "loss": 0.6467,
+      "step": 1280
+    },
+    {
+      "epoch": 1.1466666666666667,
+      "grad_norm": 1.7601350100326427,
+      "learning_rate": 3.882602784008327e-06,
+      "loss": 0.6068,
+      "step": 1290
+    },
+    {
+      "epoch": 1.1555555555555554,
+      "grad_norm": 1.7206913356738147,
+      "learning_rate": 3.8609830116947596e-06,
+      "loss": 0.6344,
+      "step": 1300
+    },
+    {
+      "epoch": 1.1555555555555554,
+      "eval_loss": 0.7862110137939453,
+      "eval_runtime": 37.9389,
+      "eval_samples_per_second": 52.716,
+      "eval_steps_per_second": 6.59,
+      "step": 1300
+    },
+    {
+      "epoch": 1.1644444444444444,
+      "grad_norm": 1.6860528463901245,
+      "learning_rate": 3.839217606650426e-06,
+      "loss": 0.6412,
+      "step": 1310
+    },
+    {
+      "epoch": 1.1733333333333333,
+      "grad_norm": 1.8854670032384495,
+      "learning_rate": 3.817308897894387e-06,
+      "loss": 0.6335,
+      "step": 1320
+    },
+    {
+      "epoch": 1.1822222222222223,
+      "grad_norm": 1.728477549429402,
+      "learning_rate": 3.7952592297799904e-06,
+      "loss": 0.6068,
+      "step": 1330
+    },
+    {
+      "epoch": 1.1911111111111112,
+      "grad_norm": 1.6820412004849736,
+      "learning_rate": 3.7730709617440227e-06,
+      "loss": 0.6379,
+      "step": 1340
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 1.648510966848167,
+      "learning_rate": 3.750746468054227e-06,
+      "loss": 0.625,
+      "step": 1350
+    },
+    {
+      "epoch": 1.208888888888889,
+      "grad_norm": 1.548799904758873,
+      "learning_rate": 3.7282881375552475e-06,
+      "loss": 0.622,
+      "step": 1360
+    },
+    {
+      "epoch": 1.2177777777777778,
+      "grad_norm": 1.6536705835515702,
+      "learning_rate": 3.70569837341301e-06,
+      "loss": 0.6554,
+      "step": 1370
+    },
+    {
+      "epoch": 1.2266666666666666,
+      "grad_norm": 1.6426851875006678,
+      "learning_rate": 3.6829795928575703e-06,
+      "loss": 0.6234,
+      "step": 1380
+    },
+    {
+      "epoch": 1.2355555555555555,
+      "grad_norm": 1.6903272164553882,
+      "learning_rate": 3.6601342269244528e-06,
+      "loss": 0.6482,
+      "step": 1390
+    },
+    {
+      "epoch": 1.2444444444444445,
+      "grad_norm": 1.86253547762314,
+      "learning_rate": 3.6371647201945216e-06,
+      "loss": 0.6181,
+      "step": 1400
+    },
+    {
+      "epoch": 1.2444444444444445,
+      "eval_loss": 0.7869328856468201,
+      "eval_runtime": 38.111,
+      "eval_samples_per_second": 52.478,
+      "eval_steps_per_second": 6.56,
+      "step": 1400
+    },
+    {
+      "epoch": 1.2533333333333334,
+      "grad_norm": 1.6989175526314024,
+      "learning_rate": 3.6140735305323943e-06,
+      "loss": 0.6278,
+      "step": 1410
+    },
+    {
+      "epoch": 1.2622222222222224,
+      "grad_norm": 1.8101109025301771,
+      "learning_rate": 3.5908631288234374e-06,
+      "loss": 0.6706,
+      "step": 1420
+    },
+    {
+      "epoch": 1.271111111111111,
+      "grad_norm": 1.9867263313582206,
+      "learning_rate": 3.5675359987093665e-06,
+      "loss": 0.6385,
+      "step": 1430
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 1.7574128013903643,
+      "learning_rate": 3.5440946363224855e-06,
+      "loss": 0.6301,
+      "step": 1440
+    },
+    {
+      "epoch": 1.2888888888888888,
+      "grad_norm": 1.4549225489268687,
+      "learning_rate": 3.5205415500185836e-06,
+      "loss": 0.6311,
+      "step": 1450
+    },
+    {
+      "epoch": 1.2977777777777777,
+      "grad_norm": 1.6494499992222607,
+      "learning_rate": 3.4968792601085296e-06,
+      "loss": 0.6596,
+      "step": 1460
+    },
+    {
+      "epoch": 1.3066666666666666,
+      "grad_norm": 1.8893405889710921,
+      "learning_rate": 3.473110298588584e-06,
+      "loss": 0.6493,
+      "step": 1470
+    },
+    {
+      "epoch": 1.3155555555555556,
+      "grad_norm": 1.6296063762923343,
+      "learning_rate": 3.4492372088694605e-06,
+      "loss": 0.6688,
+      "step": 1480
+    },
+    {
+      "epoch": 1.3244444444444445,
+      "grad_norm": 1.8187009493229125,
+      "learning_rate": 3.4252625455041684e-06,
+      "loss": 0.6553,
+      "step": 1490
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 1.6186019742868865,
+      "learning_rate": 3.4011888739146587e-06,
+      "loss": 0.6511,
+      "step": 1500
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "eval_loss": 0.7797905206680298,
+      "eval_runtime": 38.0768,
+      "eval_samples_per_second": 52.525,
+      "eval_steps_per_second": 6.566,
+      "step": 1500
+    },
+    {
+      "epoch": 1.3422222222222222,
+      "grad_norm": 1.5328967371738955,
+      "learning_rate": 3.377018770117315e-06,
+      "loss": 0.6541,
+      "step": 1510
+    },
+    {
+      "epoch": 1.3511111111111112,
+      "grad_norm": 1.7168181133470535,
+      "learning_rate": 3.3527548204472985e-06,
+      "loss": 0.6492,
+      "step": 1520
+    },
+    {
+      "epoch": 1.3599999999999999,
+      "grad_norm": 1.82949982113254,
+      "learning_rate": 3.3283996212818015e-06,
+      "loss": 0.6125,
+      "step": 1530
+    },
+    {
+      "epoch": 1.3688888888888888,
+      "grad_norm": 1.967299918797367,
+      "learning_rate": 3.303955778762217e-06,
+      "loss": 0.6609,
+      "step": 1540
+    },
+    {
+      "epoch": 1.3777777777777778,
+      "grad_norm": 1.7517637260410819,
+      "learning_rate": 3.2794259085152703e-06,
+      "loss": 0.6451,
+      "step": 1550
+    },
+    {
+      "epoch": 1.3866666666666667,
+      "grad_norm": 1.7338666575605146,
+      "learning_rate": 3.254812635373128e-06,
+      "loss": 0.6064,
+      "step": 1560
+    },
+    {
+      "epoch": 1.3955555555555557,
+      "grad_norm": 1.805258582558589,
+      "learning_rate": 3.2301185930925318e-06,
+      "loss": 0.6349,
+      "step": 1570
+    },
+    {
+      "epoch": 1.4044444444444444,
+      "grad_norm": 2.0099902327429495,
+      "learning_rate": 3.205346424072967e-06,
+      "loss": 0.6354,
+      "step": 1580
+    },
+    {
+      "epoch": 1.4133333333333333,
+      "grad_norm": 1.6968495111035833,
+      "learning_rate": 3.180498779073915e-06,
+      "loss": 0.6008,
+      "step": 1590
+    },
+    {
+      "epoch": 1.4222222222222223,
+      "grad_norm": 1.8666501019542727,
+      "learning_rate": 3.1555783169312048e-06,
+      "loss": 0.6341,
+      "step": 1600
+    },
+    {
+      "epoch": 1.4222222222222223,
+      "eval_loss": 0.7812256217002869,
+      "eval_runtime": 37.9688,
+      "eval_samples_per_second": 52.675,
+      "eval_steps_per_second": 6.584,
+      "step": 1600
+    },
+    {
+      "epoch": 1.431111111111111,
+      "grad_norm": 1.7447699710116413,
+      "learning_rate": 3.1305877042725036e-06,
+      "loss": 0.6548,
+      "step": 1610
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 1.6795897002264613,
+      "learning_rate": 3.1055296152319732e-06,
+      "loss": 0.6442,
+      "step": 1620
+    },
+    {
+      "epoch": 1.448888888888889,
+      "grad_norm": 1.8760743794492893,
+      "learning_rate": 3.0804067311641217e-06,
+      "loss": 0.66,
+      "step": 1630
+    },
+    {
+      "epoch": 1.4577777777777778,
+      "grad_norm": 1.9884017264018043,
+      "learning_rate": 3.0552217403568855e-06,
+      "loss": 0.6269,
+      "step": 1640
+    },
+    {
+      "epoch": 1.4666666666666668,
+      "grad_norm": 1.8411666706892968,
+      "learning_rate": 3.0299773377439677e-06,
+      "loss": 0.6277,
+      "step": 1650
+    },
+    {
+      "epoch": 1.4755555555555555,
+      "grad_norm": 1.5184757968572182,
+      "learning_rate": 3.0046762246164608e-06,
+      "loss": 0.6382,
+      "step": 1660
+    },
+    {
+      "epoch": 1.4844444444444445,
+      "grad_norm": 1.9260639142140834,
+      "learning_rate": 2.979321108333799e-06,
+      "loss": 0.6097,
+      "step": 1670
+    },
+    {
+      "epoch": 1.4933333333333334,
+      "grad_norm": 1.741956992068052,
+      "learning_rate": 2.953914702034054e-06,
+      "loss": 0.6319,
+      "step": 1680
+    },
+    {
+      "epoch": 1.5022222222222221,
+      "grad_norm": 1.595093207157502,
+      "learning_rate": 2.928459724343613e-06,
+      "loss": 0.6747,
+      "step": 1690
+    },
+    {
+      "epoch": 1.511111111111111,
+      "grad_norm": 1.7200717117012048,
+      "learning_rate": 2.9029588990862717e-06,
+      "loss": 0.6537,
+      "step": 1700
+    },
+    {
+      "epoch": 1.511111111111111,
+      "eval_loss": 0.7793934941291809,
+      "eval_runtime": 38.0687,
+      "eval_samples_per_second": 52.537,
+      "eval_steps_per_second": 6.567,
+      "step": 1700
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 1.6792977916911906,
+      "learning_rate": 2.8774149549917697e-06,
+      "loss": 0.6332,
+      "step": 1710
+    },
+    {
+      "epoch": 1.528888888888889,
+      "grad_norm": 1.7336128902509544,
+      "learning_rate": 2.8518306254037996e-06,
+      "loss": 0.6462,
+      "step": 1720
+    },
+    {
+      "epoch": 1.537777777777778,
+      "grad_norm": 1.6738874310796588,
+      "learning_rate": 2.82620864798753e-06,
+      "loss": 0.6446,
+      "step": 1730
+    },
+    {
+      "epoch": 1.5466666666666666,
+      "grad_norm": 1.5853846000954395,
+      "learning_rate": 2.800551764436652e-06,
+      "loss": 0.6864,
+      "step": 1740
+    },
+    {
+      "epoch": 1.5555555555555556,
+      "grad_norm": 1.7102684215364048,
+      "learning_rate": 2.774862720180008e-06,
+      "loss": 0.6244,
+      "step": 1750
+    },
+    {
+      "epoch": 1.5644444444444443,
+      "grad_norm": 1.6441239452770726,
+      "learning_rate": 2.749144264087814e-06,
+      "loss": 0.6183,
+      "step": 1760
+    },
+    {
+      "epoch": 1.5733333333333333,
+      "grad_norm": 1.8848908060309273,
+      "learning_rate": 2.7233991481775173e-06,
+      "loss": 0.6524,
+      "step": 1770
+    },
+    {
+      "epoch": 1.5822222222222222,
+      "grad_norm": 1.5767138461588817,
+      "learning_rate": 2.697630127319312e-06,
+      "loss": 0.6415,
+      "step": 1780
+    },
+    {
+      "epoch": 1.5911111111111111,
+      "grad_norm": 1.6578116610765519,
+      "learning_rate": 2.6718399589413533e-06,
+      "loss": 0.6488,
+      "step": 1790
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 1.6463146375379087,
+      "learning_rate": 2.6460314027347002e-06,
+      "loss": 0.6626,
+      "step": 1800
+    },
+    {
+      "epoch": 1.6,
+      "eval_loss": 0.7779668569564819,
+      "eval_runtime": 38.0224,
+      "eval_samples_per_second": 52.601,
+      "eval_steps_per_second": 6.575,
+      "step": 1800
+    },
+    {
+      "epoch": 1.608888888888889,
+      "grad_norm": 1.3794992178377827,
+      "learning_rate": 2.6202072203580098e-06,
+      "loss": 0.6213,
+      "step": 1810
+    },
+    {
+      "epoch": 1.6177777777777778,
+      "grad_norm": 1.6547672801771742,
+      "learning_rate": 2.594370175142029e-06,
+      "loss": 0.622,
+      "step": 1820
+    },
+    {
+      "epoch": 1.6266666666666667,
+      "grad_norm": 1.8131058123768224,
+      "learning_rate": 2.5685230317938946e-06,
+      "loss": 0.6115,
+      "step": 1830
+    },
+    {
+      "epoch": 1.6355555555555554,
+      "grad_norm": 1.5043497534471073,
+      "learning_rate": 2.542668556101305e-06,
+      "loss": 0.6278,
+      "step": 1840
+    },
+    {
+      "epoch": 1.6444444444444444,
+      "grad_norm": 1.6388736348728028,
+      "learning_rate": 2.516809514636556e-06,
+      "loss": 0.6419,
+      "step": 1850
+    },
+    {
+      "epoch": 1.6533333333333333,
+      "grad_norm": 1.6660885474749705,
+      "learning_rate": 2.4909486744605105e-06,
+      "loss": 0.6158,
+      "step": 1860
+    },
+    {
+      "epoch": 1.6622222222222223,
+      "grad_norm": 1.7074374325187731,
+      "learning_rate": 2.4650888028264993e-06,
+      "loss": 0.6404,
+      "step": 1870
+    },
+    {
+      "epoch": 1.6711111111111112,
+      "grad_norm": 1.5200201087278384,
+      "learning_rate": 2.439232666884216e-06,
+      "loss": 0.6334,
+      "step": 1880
+    },
+    {
+      "epoch": 1.6800000000000002,
+      "grad_norm": 1.7294919605993466,
+      "learning_rate": 2.413383033383614e-06,
+      "loss": 0.6486,
+      "step": 1890
+    },
+    {
+      "epoch": 1.6888888888888889,
+      "grad_norm": 1.6336612869980456,
+      "learning_rate": 2.3875426683788497e-06,
+      "loss": 0.6116,
+      "step": 1900
+    },
+    {
+      "epoch": 1.6888888888888889,
+      "eval_loss": 0.7765544056892395,
+      "eval_runtime": 37.883,
+      "eval_samples_per_second": 52.794,
+      "eval_steps_per_second": 6.599,
+      "step": 1900
+    },
+    {
+      "epoch": 1.6977777777777778,
+      "grad_norm": 1.709115790338184,
+      "learning_rate": 2.3617143369322988e-06,
+      "loss": 0.6252,
+      "step": 1910
+    },
+    {
+      "epoch": 1.7066666666666666,
+      "grad_norm": 1.7821417246615834,
+      "learning_rate": 2.33590080281868e-06,
+      "loss": 0.6107,
+      "step": 1920
+    },
+    {
+      "epoch": 1.7155555555555555,
+      "grad_norm": 1.6445009506847397,
+      "learning_rate": 2.310104828229313e-06,
+      "loss": 0.6142,
+      "step": 1930
+    },
+    {
+      "epoch": 1.7244444444444444,
+      "grad_norm": 1.841236220247559,
+      "learning_rate": 2.2843291734765544e-06,
+      "loss": 0.6502,
+      "step": 1940
+    },
+    {
+      "epoch": 1.7333333333333334,
+      "grad_norm": 1.842694284367171,
+      "learning_rate": 2.2585765966984236e-06,
+      "loss": 0.674,
+      "step": 1950
+    },
+    {
+      "epoch": 1.7422222222222223,
+      "grad_norm": 1.8090827021490723,
+      "learning_rate": 2.2328498535634704e-06,
+      "loss": 0.6175,
+      "step": 1960
+    },
+    {
+      "epoch": 1.751111111111111,
+      "grad_norm": 1.6878272228854192,
+      "learning_rate": 2.2071516969758988e-06,
+      "loss": 0.6231,
+      "step": 1970
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 1.9731653412399757,
+      "learning_rate": 2.181484876780996e-06,
+      "loss": 0.629,
+      "step": 1980
+    },
+    {
+      "epoch": 1.7688888888888887,
+      "grad_norm": 1.6667636315286483,
+      "learning_rate": 2.1558521394708793e-06,
+      "loss": 0.6658,
+      "step": 1990
+    },
+    {
+      "epoch": 1.7777777777777777,
+      "grad_norm": 1.7254699126772677,
+      "learning_rate": 2.1302562278906106e-06,
+      "loss": 0.6327,
+      "step": 2000
+    },
+    {
+      "epoch": 1.7777777777777777,
+      "eval_loss": 0.7731255292892456,
+      "eval_runtime": 38.4418,
+      "eval_samples_per_second": 52.027,
+      "eval_steps_per_second": 6.503,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3375,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 102798860353536.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12abd2c91b01a822d41c856433df3965960b85daf3f009e4554cb8f02f3ee956
+size 7160

checkpoint-2000/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

checkpoint-3000/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoint-3000/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.1",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

checkpoint-3000/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.46.1"
+}

checkpoint-3000/global_step3000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a76883af89258bb455bb0b4e591506c4e0a0489e308448b2b42f24d193296f2
+size 11423430028