Add files using upload-large-folder tool

config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "Emu3ForCausalLM"
+  ],
+  "attention_dropout": 0.1,
+  "auto_map": {
+    "AutoConfig": "BAAI/Emu3-Stage1--configuration_emu3.Emu3Config",
+    "AutoModelForCausalLM": "BAAI/Emu3-Stage1--modeling_emu3.Emu3ForCausalLM"
+  },
+  "boi_token_id": 151852,
+  "bos_token_id": 151849,
+  "eof_token_id": 151847,
+  "eoi_token_id": 151853,
+  "eol_token_id": 151846,
+  "eos_token_id": 151850,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "image_area": 65536,
+  "img_token_id": 151851,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 4300,
+  "model_type": "Emu3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 151643,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": false,
+  "vocab_size": 184622
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151849,
+  "eos_token_id": 151850,
+  "pad_token_id": 151643,
+  "transformers_version": "4.51.3"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bcf7ee3ce4748103b6f82cd71aacc9afddf3d2ed332d7e902b6f1c5cfc0cbd8
+size 4884766656

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:170bf2fc78db49c910245e236ca318210d897be320a6ae89fad10ff5038180c4
+size 4999819320

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:044e446c4bfd1cf59b5419ce627c2d857650737ef280ef0e8dfa34b6a2500838
+size 4915916184

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69896f7c15d49e10155971654a23ac5c695e2366c764a8135ddb693b81897aa4
+size 2183554760

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16984023040
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a8fd8f8fa25dc9c72ffe5c33017ac5bd1c7f0e3eab2ffa0afd2a70dfb67181a
+size 15984

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6506b9dac8338674b040d0855b6626c7912ca1febae648f81a6d00bddc22b59
+size 15984

rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:802fd73a5ba8c9b04ca3ff466d5f6c33709ceadbfd94e5be69773677565f60b4
+size 15984

rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eccedd7855b66a982c31c5faaa21b552862e3b52da2db1c07e7151d7d433533c
+size 15984

rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:381af05376596ba601d462616703e8e9f83ef675a567988f3675a2493e7c5762
+size 15984

rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8457eab8e7f7ec705a74ddf5df53ebbdf0b137549fbeb9032711b54b076d32f5
+size 15984

rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02cc7f48ad42c221aae2dec8d063b6b29c574a23a3d8a6896b76ae9842672415
+size 15984

rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2be3b63e5e7b67b8fe620dce983474903e105bc6bb95efff0690e65f0e7ab4d8
+size 15984

trainer_state.json

@@ -0,0 +1,1266 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9496127404055348,
+  "eval_steps": 100,
+  "global_step": 5600,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.017404925593943087,
+      "grad_norm": 4.2433902126025425,
+      "learning_rate": 9.8e-05,
+      "loss": 3.6204,
+      "step": 50
+    },
+    {
+      "epoch": 0.034809851187886175,
+      "grad_norm": 3.0786203091123565,
+      "learning_rate": 9.999928647255986e-05,
+      "loss": 2.5602,
+      "step": 100
+    },
+    {
+      "epoch": 0.034809851187886175,
+      "eval_loss": 2.363542079925537,
+      "eval_runtime": 14.092,
+      "eval_samples_per_second": 70.962,
+      "eval_steps_per_second": 2.271,
+      "step": 100
+    },
+    {
+      "epoch": 0.05221477678182926,
+      "grad_norm": 2.489798826862787,
+      "learning_rate": 9.999708736748881e-05,
+      "loss": 2.2577,
+      "step": 150
+    },
+    {
+      "epoch": 0.06961970237577235,
+      "grad_norm": 4.227397206704295,
+      "learning_rate": 9.999340245361986e-05,
+      "loss": 2.065,
+      "step": 200
+    },
+    {
+      "epoch": 0.06961970237577235,
+      "eval_loss": 2.0204052925109863,
+      "eval_runtime": 14.0707,
+      "eval_samples_per_second": 71.07,
+      "eval_steps_per_second": 2.274,
+      "step": 200
+    },
+    {
+      "epoch": 0.08702462796971543,
+      "grad_norm": 1.5627603609182088,
+      "learning_rate": 9.998823184156712e-05,
+      "loss": 1.9504,
+      "step": 250
+    },
+    {
+      "epoch": 0.10442955356365852,
+      "grad_norm": 2.0849274528531834,
+      "learning_rate": 9.998157568654259e-05,
+      "loss": 1.9106,
+      "step": 300
+    },
+    {
+      "epoch": 0.10442955356365852,
+      "eval_loss": 1.8868601322174072,
+      "eval_runtime": 14.0214,
+      "eval_samples_per_second": 71.319,
+      "eval_steps_per_second": 2.282,
+      "step": 300
+    },
+    {
+      "epoch": 0.12183447915760161,
+      "grad_norm": 1.2760571472974125,
+      "learning_rate": 9.997343418835142e-05,
+      "loss": 1.8861,
+      "step": 350
+    },
+    {
+      "epoch": 0.1392394047515447,
+      "grad_norm": 3.7430921365305005,
+      "learning_rate": 9.996380759138595e-05,
+      "loss": 1.8622,
+      "step": 400
+    },
+    {
+      "epoch": 0.1392394047515447,
+      "eval_loss": 1.8383088111877441,
+      "eval_runtime": 14.0403,
+      "eval_samples_per_second": 71.224,
+      "eval_steps_per_second": 2.279,
+      "step": 400
+    },
+    {
+      "epoch": 0.15664433034548778,
+      "grad_norm": 0.9586199143769504,
+      "learning_rate": 9.995269618461844e-05,
+      "loss": 1.8478,
+      "step": 450
+    },
+    {
+      "epoch": 0.17404925593943085,
+      "grad_norm": 1.5061363604288809,
+      "learning_rate": 9.99401003015922e-05,
+      "loss": 1.8117,
+      "step": 500
+    },
+    {
+      "epoch": 0.17404925593943085,
+      "eval_loss": 1.7942754030227661,
+      "eval_runtime": 14.0551,
+      "eval_samples_per_second": 71.149,
+      "eval_steps_per_second": 2.277,
+      "step": 500
+    },
+    {
+      "epoch": 0.19145418153337396,
+      "grad_norm": 2.433096036143497,
+      "learning_rate": 9.992602032041181e-05,
+      "loss": 1.8071,
+      "step": 550
+    },
+    {
+      "epoch": 0.20885910712731703,
+      "grad_norm": 3.0865325369733796,
+      "learning_rate": 9.991045666373163e-05,
+      "loss": 1.7895,
+      "step": 600
+    },
+    {
+      "epoch": 0.20885910712731703,
+      "eval_loss": 1.7960834503173828,
+      "eval_runtime": 14.0575,
+      "eval_samples_per_second": 71.136,
+      "eval_steps_per_second": 2.276,
+      "step": 600
+    },
+    {
+      "epoch": 0.2262640327212601,
+      "grad_norm": 1.0844936845986464,
+      "learning_rate": 9.989340979874317e-05,
+      "loss": 1.7954,
+      "step": 650
+    },
+    {
+      "epoch": 0.24366895831520322,
+      "grad_norm": 1.5841141135500758,
+      "learning_rate": 9.987488023716102e-05,
+      "loss": 1.7827,
+      "step": 700
+    },
+    {
+      "epoch": 0.24366895831520322,
+      "eval_loss": 1.7635940313339233,
+      "eval_runtime": 14.0192,
+      "eval_samples_per_second": 71.331,
+      "eval_steps_per_second": 2.283,
+      "step": 700
+    },
+    {
+      "epoch": 0.26107388390914626,
+      "grad_norm": 1.0358659820491174,
+      "learning_rate": 9.985486853520748e-05,
+      "loss": 1.7755,
+      "step": 750
+    },
+    {
+      "epoch": 0.2784788095030894,
+      "grad_norm": 2.921637768400008,
+      "learning_rate": 9.983337529359597e-05,
+      "loss": 1.7689,
+      "step": 800
+    },
+    {
+      "epoch": 0.2784788095030894,
+      "eval_loss": 1.7601885795593262,
+      "eval_runtime": 14.0539,
+      "eval_samples_per_second": 71.155,
+      "eval_steps_per_second": 2.277,
+      "step": 800
+    },
+    {
+      "epoch": 0.2958837350970325,
+      "grad_norm": 1.2109225200496343,
+      "learning_rate": 9.981040115751287e-05,
+      "loss": 1.7642,
+      "step": 850
+    },
+    {
+      "epoch": 0.31328866069097555,
+      "grad_norm": 0.9168156411457995,
+      "learning_rate": 9.978594681659822e-05,
+      "loss": 1.7584,
+      "step": 900
+    },
+    {
+      "epoch": 0.31328866069097555,
+      "eval_loss": 1.733883023262024,
+      "eval_runtime": 14.0773,
+      "eval_samples_per_second": 71.036,
+      "eval_steps_per_second": 2.273,
+      "step": 900
+    },
+    {
+      "epoch": 0.33069358628491863,
+      "grad_norm": 1.231514474527335,
+      "learning_rate": 9.976001300492505e-05,
+      "loss": 1.7476,
+      "step": 950
+    },
+    {
+      "epoch": 0.3480985118788617,
+      "grad_norm": 1.3345795373547282,
+      "learning_rate": 9.97326005009772e-05,
+      "loss": 1.7529,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3480985118788617,
+      "eval_loss": 1.7277562618255615,
+      "eval_runtime": 14.0558,
+      "eval_samples_per_second": 71.145,
+      "eval_steps_per_second": 2.277,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3655034374728048,
+      "grad_norm": 1.3848562250830305,
+      "learning_rate": 9.970371012762615e-05,
+      "loss": 1.7383,
+      "step": 1050
+    },
+    {
+      "epoch": 0.3829083630667479,
+      "grad_norm": 0.9102870237217918,
+      "learning_rate": 9.967334275210616e-05,
+      "loss": 1.7312,
+      "step": 1100
+    },
+    {
+      "epoch": 0.3829083630667479,
+      "eval_loss": 1.7197346687316895,
+      "eval_runtime": 14.1392,
+      "eval_samples_per_second": 70.725,
+      "eval_steps_per_second": 2.263,
+      "step": 1100
+    },
+    {
+      "epoch": 0.400313288660691,
+      "grad_norm": 0.9403631790293067,
+      "learning_rate": 9.964149928598834e-05,
+      "loss": 1.7354,
+      "step": 1150
+    },
+    {
+      "epoch": 0.41771821425463407,
+      "grad_norm": 1.5743784967989016,
+      "learning_rate": 9.96081806851532e-05,
+      "loss": 1.7384,
+      "step": 1200
+    },
+    {
+      "epoch": 0.41771821425463407,
+      "eval_loss": 1.7184182405471802,
+      "eval_runtime": 14.0812,
+      "eval_samples_per_second": 71.017,
+      "eval_steps_per_second": 2.273,
+      "step": 1200
+    },
+    {
+      "epoch": 0.43512313984857715,
+      "grad_norm": 2.0859941497191743,
+      "learning_rate": 9.957338794976201e-05,
+      "loss": 1.7389,
+      "step": 1250
+    },
+    {
+      "epoch": 0.4525280654425202,
+      "grad_norm": 1.347628827751819,
+      "learning_rate": 9.953712212422681e-05,
+      "loss": 1.7267,
+      "step": 1300
+    },
+    {
+      "epoch": 0.4525280654425202,
+      "eval_loss": 1.7116867303848267,
+      "eval_runtime": 14.0496,
+      "eval_samples_per_second": 71.177,
+      "eval_steps_per_second": 2.278,
+      "step": 1300
+    },
+    {
+      "epoch": 0.4699329910364633,
+      "grad_norm": 0.7775683989797394,
+      "learning_rate": 9.949938429717895e-05,
+      "loss": 1.7136,
+      "step": 1350
+    },
+    {
+      "epoch": 0.48733791663040643,
+      "grad_norm": 0.7955090270012505,
+      "learning_rate": 9.946017560143651e-05,
+      "loss": 1.7188,
+      "step": 1400
+    },
+    {
+      "epoch": 0.48733791663040643,
+      "eval_loss": 1.703679084777832,
+      "eval_runtime": 14.077,
+      "eval_samples_per_second": 71.038,
+      "eval_steps_per_second": 2.273,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5047428422243495,
+      "grad_norm": 1.2332760111187542,
+      "learning_rate": 9.941949721397028e-05,
+      "loss": 1.7169,
+      "step": 1450
+    },
+    {
+      "epoch": 0.5221477678182925,
+      "grad_norm": 1.81355607393607,
+      "learning_rate": 9.93773503558684e-05,
+      "loss": 1.7157,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5221477678182925,
+      "eval_loss": 1.7078830003738403,
+      "eval_runtime": 14.09,
+      "eval_samples_per_second": 70.972,
+      "eval_steps_per_second": 2.271,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5395526934122357,
+      "grad_norm": 0.9053542478773059,
+      "learning_rate": 9.933373629229969e-05,
+      "loss": 1.7102,
+      "step": 1550
+    },
+    {
+      "epoch": 0.5569576190061788,
+      "grad_norm": 0.6503277295238644,
+      "learning_rate": 9.928865633247573e-05,
+      "loss": 1.7033,
+      "step": 1600
+    },
+    {
+      "epoch": 0.5569576190061788,
+      "eval_loss": 1.6917779445648193,
+      "eval_runtime": 14.0698,
+      "eval_samples_per_second": 71.074,
+      "eval_steps_per_second": 2.274,
+      "step": 1600
+    },
+    {
+      "epoch": 0.5743625446001218,
+      "grad_norm": 1.4224490096345375,
+      "learning_rate": 9.92421118296115e-05,
+      "loss": 1.6997,
+      "step": 1650
+    },
+    {
+      "epoch": 0.591767470194065,
+      "grad_norm": 0.7864420926166752,
+      "learning_rate": 9.919410418088481e-05,
+      "loss": 1.7102,
+      "step": 1700
+    },
+    {
+      "epoch": 0.591767470194065,
+      "eval_loss": 1.690305233001709,
+      "eval_runtime": 14.1062,
+      "eval_samples_per_second": 70.891,
+      "eval_steps_per_second": 2.269,
+      "step": 1700
+    },
+    {
+      "epoch": 0.609172395788008,
+      "grad_norm": 0.5663590518834145,
+      "learning_rate": 9.914463482739435e-05,
+      "loss": 1.7046,
+      "step": 1750
+    },
+    {
+      "epoch": 0.6265773213819511,
+      "grad_norm": 1.1145025421986445,
+      "learning_rate": 9.909370525411637e-05,
+      "loss": 1.6905,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6265773213819511,
+      "eval_loss": 1.6856919527053833,
+      "eval_runtime": 14.0345,
+      "eval_samples_per_second": 71.253,
+      "eval_steps_per_second": 2.28,
+      "step": 1800
+    },
+    {
+      "epoch": 0.6439822469758941,
+      "grad_norm": 1.079593642429848,
+      "learning_rate": 9.90413169898602e-05,
+      "loss": 1.6973,
+      "step": 1850
+    },
+    {
+      "epoch": 0.6613871725698373,
+      "grad_norm": 0.8794305699903086,
+      "learning_rate": 9.898747160722229e-05,
+      "loss": 1.6821,
+      "step": 1900
+    },
+    {
+      "epoch": 0.6613871725698373,
+      "eval_loss": 1.680002212524414,
+      "eval_runtime": 14.0923,
+      "eval_samples_per_second": 70.961,
+      "eval_steps_per_second": 2.271,
+      "step": 1900
+    },
+    {
+      "epoch": 0.6787920981637804,
+      "grad_norm": 1.3664190261530837,
+      "learning_rate": 9.893217072253903e-05,
+      "loss": 1.6909,
+      "step": 1950
+    },
+    {
+      "epoch": 0.6961970237577234,
+      "grad_norm": 0.9268231360918758,
+      "learning_rate": 9.88754159958382e-05,
+      "loss": 1.6901,
+      "step": 2000
+    },
+    {
+      "epoch": 0.6961970237577234,
+      "eval_loss": 1.6765377521514893,
+      "eval_runtime": 14.0942,
+      "eval_samples_per_second": 70.951,
+      "eval_steps_per_second": 2.27,
+      "step": 2000
+    },
+    {
+      "epoch": 0.7136019493516665,
+      "grad_norm": 0.9864416812238661,
+      "learning_rate": 9.881720913078921e-05,
+      "loss": 1.6911,
+      "step": 2050
+    },
+    {
+      "epoch": 0.7310068749456096,
+      "grad_norm": 0.8706035984933645,
+      "learning_rate": 9.875755187465186e-05,
+      "loss": 1.6866,
+      "step": 2100
+    },
+    {
+      "epoch": 0.7310068749456096,
+      "eval_loss": 1.675471305847168,
+      "eval_runtime": 14.0392,
+      "eval_samples_per_second": 71.229,
+      "eval_steps_per_second": 2.279,
+      "step": 2100
+    },
+    {
+      "epoch": 0.7484118005395527,
+      "grad_norm": 0.9954026204157976,
+      "learning_rate": 9.869644601822396e-05,
+      "loss": 1.6764,
+      "step": 2150
+    },
+    {
+      "epoch": 0.7658167261334958,
+      "grad_norm": 0.9859776473729975,
+      "learning_rate": 9.863389339578761e-05,
+      "loss": 1.6772,
+      "step": 2200
+    },
+    {
+      "epoch": 0.7658167261334958,
+      "eval_loss": 1.6698520183563232,
+      "eval_runtime": 14.0605,
+      "eval_samples_per_second": 71.121,
+      "eval_steps_per_second": 2.276,
+      "step": 2200
+    },
+    {
+      "epoch": 0.7832216517274389,
+      "grad_norm": 0.9106273220771831,
+      "learning_rate": 9.856989588505399e-05,
+      "loss": 1.6796,
+      "step": 2250
+    },
+    {
+      "epoch": 0.800626577321382,
+      "grad_norm": 1.1219788313484198,
+      "learning_rate": 9.850445540710714e-05,
+      "loss": 1.6742,
+      "step": 2300
+    },
+    {
+      "epoch": 0.800626577321382,
+      "eval_loss": 1.663262963294983,
+      "eval_runtime": 14.0171,
+      "eval_samples_per_second": 71.341,
+      "eval_steps_per_second": 2.283,
+      "step": 2300
+    },
+    {
+      "epoch": 0.818031502915325,
+      "grad_norm": 0.7584108888049894,
+      "learning_rate": 9.843757392634629e-05,
+      "loss": 1.6773,
+      "step": 2350
+    },
+    {
+      "epoch": 0.8354364285092681,
+      "grad_norm": 0.8524680066268957,
+      "learning_rate": 9.836925345042675e-05,
+      "loss": 1.6802,
+      "step": 2400
+    },
+    {
+      "epoch": 0.8354364285092681,
+      "eval_loss": 1.6637836694717407,
+      "eval_runtime": 14.0523,
+      "eval_samples_per_second": 71.163,
+      "eval_steps_per_second": 2.277,
+      "step": 2400
+    },
+    {
+      "epoch": 0.8528413541032112,
+      "grad_norm": 0.5830225213698085,
+      "learning_rate": 9.82994960301998e-05,
+      "loss": 1.6774,
+      "step": 2450
+    },
+    {
+      "epoch": 0.8702462796971543,
+      "grad_norm": 0.8294538008262156,
+      "learning_rate": 9.822830375965103e-05,
+      "loss": 1.6702,
+      "step": 2500
+    },
+    {
+      "epoch": 0.8702462796971543,
+      "eval_loss": 1.6600449085235596,
+      "eval_runtime": 14.0902,
+      "eval_samples_per_second": 70.972,
+      "eval_steps_per_second": 2.271,
+      "step": 2500
+    },
+    {
+      "epoch": 0.8876512052910974,
+      "grad_norm": 1.0515224700476833,
+      "learning_rate": 9.815567877583758e-05,
+      "loss": 1.6758,
+      "step": 2550
+    },
+    {
+      "epoch": 0.9050561308850404,
+      "grad_norm": 0.7866141842693181,
+      "learning_rate": 9.808162325882385e-05,
+      "loss": 1.6645,
+      "step": 2600
+    },
+    {
+      "epoch": 0.9050561308850404,
+      "eval_loss": 1.657778263092041,
+      "eval_runtime": 14.0817,
+      "eval_samples_per_second": 71.014,
+      "eval_steps_per_second": 2.272,
+      "step": 2600
+    },
+    {
+      "epoch": 0.9224610564789836,
+      "grad_norm": 0.9909101251943951,
+      "learning_rate": 9.800613943161619e-05,
+      "loss": 1.6629,
+      "step": 2650
+    },
+    {
+      "epoch": 0.9398659820729266,
+      "grad_norm": 0.9534991636209588,
+      "learning_rate": 9.79292295600961e-05,
+      "loss": 1.6523,
+      "step": 2700
+    },
+    {
+      "epoch": 0.9398659820729266,
+      "eval_loss": 1.6576528549194336,
+      "eval_runtime": 14.0341,
+      "eval_samples_per_second": 71.255,
+      "eval_steps_per_second": 2.28,
+      "step": 2700
+    },
+    {
+      "epoch": 0.9572709076668697,
+      "grad_norm": 0.7620610436511178,
+      "learning_rate": 9.785089595295222e-05,
+      "loss": 1.6573,
+      "step": 2750
+    },
+    {
+      "epoch": 0.9746758332608129,
+      "grad_norm": 1.5752171211110084,
+      "learning_rate": 9.777114096161105e-05,
+      "loss": 1.6583,
+      "step": 2800
+    },
+    {
+      "epoch": 0.9746758332608129,
+      "eval_loss": 1.6622099876403809,
+      "eval_runtime": 14.0927,
+      "eval_samples_per_second": 70.959,
+      "eval_steps_per_second": 2.271,
+      "step": 2800
+    },
+    {
+      "epoch": 0.9920807588547559,
+      "grad_norm": 0.5970970379963504,
+      "learning_rate": 9.768996698016636e-05,
+      "loss": 1.6625,
+      "step": 2850
+    },
+    {
+      "epoch": 1.009746758332608,
+      "grad_norm": 0.7470976369983713,
+      "learning_rate": 9.760737644530726e-05,
+      "loss": 1.6597,
+      "step": 2900
+    },
+    {
+      "epoch": 1.009746758332608,
+      "eval_loss": 1.647603988647461,
+      "eval_runtime": 14.12,
+      "eval_samples_per_second": 70.822,
+      "eval_steps_per_second": 2.266,
+      "step": 2900
+    },
+    {
+      "epoch": 1.0271516839265513,
+      "grad_norm": 0.962160586071795,
+      "learning_rate": 9.75233718362452e-05,
+      "loss": 1.611,
+      "step": 2950
+    },
+    {
+      "epoch": 1.0445566095204943,
+      "grad_norm": 0.6386050774526276,
+      "learning_rate": 9.74379556746394e-05,
+      "loss": 1.619,
+      "step": 3000
+    },
+    {
+      "epoch": 1.0445566095204943,
+      "eval_loss": 1.6434565782546997,
+      "eval_runtime": 14.0132,
+      "eval_samples_per_second": 71.361,
+      "eval_steps_per_second": 2.284,
+      "step": 3000
+    },
+    {
+      "epoch": 1.0619615351144374,
+      "grad_norm": 1.5569952795665942,
+      "learning_rate": 9.735113052452119e-05,
+      "loss": 1.6108,
+      "step": 3050
+    },
+    {
+      "epoch": 1.0793664607083804,
+      "grad_norm": 1.223444554102184,
+      "learning_rate": 9.726289899221713e-05,
+      "loss": 1.6242,
+      "step": 3100
+    },
+    {
+      "epoch": 1.0793664607083804,
+      "eval_loss": 1.6534233093261719,
+      "eval_runtime": 14.0914,
+      "eval_samples_per_second": 70.965,
+      "eval_steps_per_second": 2.271,
+      "step": 3100
+    },
+    {
+      "epoch": 1.0967713863023236,
+      "grad_norm": 0.6055563672851731,
+      "learning_rate": 9.717326372627065e-05,
+      "loss": 1.6165,
+      "step": 3150
+    },
+    {
+      "epoch": 1.1141763118962666,
+      "grad_norm": 0.7125630072846985,
+      "learning_rate": 9.708222741736268e-05,
+      "loss": 1.6137,
+      "step": 3200
+    },
+    {
+      "epoch": 1.1141763118962666,
+      "eval_loss": 1.6405473947525024,
+      "eval_runtime": 14.0433,
+      "eval_samples_per_second": 71.208,
+      "eval_steps_per_second": 2.279,
+      "step": 3200
+    },
+    {
+      "epoch": 1.1315812374902097,
+      "grad_norm": 0.6828372843368237,
+      "learning_rate": 9.698979279823071e-05,
+      "loss": 1.6178,
+      "step": 3250
+    },
+    {
+      "epoch": 1.148986163084153,
+      "grad_norm": 0.6458088716811551,
+      "learning_rate": 9.689596264358694e-05,
+      "loss": 1.6057,
+      "step": 3300
+    },
+    {
+      "epoch": 1.148986163084153,
+      "eval_loss": 1.6405302286148071,
+      "eval_runtime": 14.0715,
+      "eval_samples_per_second": 71.065,
+      "eval_steps_per_second": 2.274,
+      "step": 3300
+    },
+    {
+      "epoch": 1.166391088678096,
+      "grad_norm": 0.7900271544609745,
+      "learning_rate": 9.680073977003483e-05,
+      "loss": 1.6031,
+      "step": 3350
+    },
+    {
+      "epoch": 1.183796014272039,
+      "grad_norm": 1.0152045530917768,
+      "learning_rate": 9.670412703598469e-05,
+      "loss": 1.6117,
+      "step": 3400
+    },
+    {
+      "epoch": 1.183796014272039,
+      "eval_loss": 1.639701247215271,
+      "eval_runtime": 14.0316,
+      "eval_samples_per_second": 71.268,
+      "eval_steps_per_second": 2.281,
+      "step": 3400
+    },
+    {
+      "epoch": 1.201200939865982,
+      "grad_norm": 0.7302281809291777,
+      "learning_rate": 9.660612734156777e-05,
+      "loss": 1.6027,
+      "step": 3450
+    },
+    {
+      "epoch": 1.2186058654599252,
+      "grad_norm": 0.6755389511951054,
+      "learning_rate": 9.650674362854923e-05,
+      "loss": 1.6227,
+      "step": 3500
+    },
+    {
+      "epoch": 1.2186058654599252,
+      "eval_loss": 1.633447289466858,
+      "eval_runtime": 14.0553,
+      "eval_samples_per_second": 71.148,
+      "eval_steps_per_second": 2.277,
+      "step": 3500
+    },
+    {
+      "epoch": 1.2360107910538682,
+      "grad_norm": 0.4958097178118167,
+      "learning_rate": 9.640597888023988e-05,
+      "loss": 1.6039,
+      "step": 3550
+    },
+    {
+      "epoch": 1.2534157166478113,
+      "grad_norm": 0.842473221384133,
+      "learning_rate": 9.630383612140661e-05,
+      "loss": 1.6105,
+      "step": 3600
+    },
+    {
+      "epoch": 1.2534157166478113,
+      "eval_loss": 1.6299790143966675,
+      "eval_runtime": 14.0661,
+      "eval_samples_per_second": 71.093,
+      "eval_steps_per_second": 2.275,
+      "step": 3600
+    },
+    {
+      "epoch": 1.2708206422417545,
+      "grad_norm": 0.5547865266670734,
+      "learning_rate": 9.62003184181815e-05,
+      "loss": 1.608,
+      "step": 3650
+    },
+    {
+      "epoch": 1.2882255678356975,
+      "grad_norm": 0.5625620088465835,
+      "learning_rate": 9.609542887796993e-05,
+      "loss": 1.6141,
+      "step": 3700
+    },
+    {
+      "epoch": 1.2882255678356975,
+      "eval_loss": 1.6269824504852295,
+      "eval_runtime": 14.0355,
+      "eval_samples_per_second": 71.248,
+      "eval_steps_per_second": 2.28,
+      "step": 3700
+    },
+    {
+      "epoch": 1.3056304934296405,
+      "grad_norm": 0.9870933875943105,
+      "learning_rate": 9.598917064935719e-05,
+      "loss": 1.6045,
+      "step": 3750
+    },
+    {
+      "epoch": 1.3230354190235838,
+      "grad_norm": 0.6538637454488698,
+      "learning_rate": 9.5881546922014e-05,
+      "loss": 1.601,
+      "step": 3800
+    },
+    {
+      "epoch": 1.3230354190235838,
+      "eval_loss": 1.626142144203186,
+      "eval_runtime": 14.1089,
+      "eval_samples_per_second": 70.877,
+      "eval_steps_per_second": 2.268,
+      "step": 3800
+    },
+    {
+      "epoch": 1.3404403446175268,
+      "grad_norm": 0.6547060258929419,
+      "learning_rate": 9.57725609266008e-05,
+      "loss": 1.6066,
+      "step": 3850
+    },
+    {
+      "epoch": 1.3578452702114698,
+      "grad_norm": 0.9358458562600437,
+      "learning_rate": 9.566221593467069e-05,
+      "loss": 1.6221,
+      "step": 3900
+    },
+    {
+      "epoch": 1.3578452702114698,
+      "eval_loss": 1.627410888671875,
+      "eval_runtime": 14.088,
+      "eval_samples_per_second": 70.982,
+      "eval_steps_per_second": 2.271,
+      "step": 3900
+    },
+    {
+      "epoch": 1.3752501958054129,
+      "grad_norm": 0.8129191474694835,
+      "learning_rate": 9.555051525857134e-05,
+      "loss": 1.5996,
+      "step": 3950
+    },
+    {
+      "epoch": 1.392655121399356,
+      "grad_norm": 0.6824919031119797,
+      "learning_rate": 9.54374622513454e-05,
+      "loss": 1.6101,
+      "step": 4000
+    },
+    {
+      "epoch": 1.392655121399356,
+      "eval_loss": 1.6165417432785034,
+      "eval_runtime": 14.0492,
+      "eval_samples_per_second": 71.179,
+      "eval_steps_per_second": 2.278,
+      "step": 4000
+    },
+    {
+      "epoch": 1.4100600469932991,
+      "grad_norm": 0.9330542502271321,
+      "learning_rate": 9.532306030663e-05,
+      "loss": 1.5958,
+      "step": 4050
+    },
+    {
+      "epoch": 1.4274649725872421,
+      "grad_norm": 0.6438330837104954,
+      "learning_rate": 9.520731285855482e-05,
+      "loss": 1.599,
+      "step": 4100
+    },
+    {
+      "epoch": 1.4274649725872421,
+      "eval_loss": 1.6210800409317017,
+      "eval_runtime": 14.0932,
+      "eval_samples_per_second": 70.956,
+      "eval_steps_per_second": 2.271,
+      "step": 4100
+    },
+    {
+      "epoch": 1.4448698981811852,
+      "grad_norm": 0.9636631744898069,
+      "learning_rate": 9.509022338163896e-05,
+      "loss": 1.5955,
+      "step": 4150
+    },
+    {
+      "epoch": 1.4622748237751284,
+      "grad_norm": 0.5569273625801461,
+      "learning_rate": 9.497179539068673e-05,
+      "loss": 1.6007,
+      "step": 4200
+    },
+    {
+      "epoch": 1.4622748237751284,
+      "eval_loss": 1.6149400472640991,
+      "eval_runtime": 14.0717,
+      "eval_samples_per_second": 71.064,
+      "eval_steps_per_second": 2.274,
+      "step": 4200
+    },
+    {
+      "epoch": 1.4796797493690714,
+      "grad_norm": 0.5160141243848255,
+      "learning_rate": 9.485203244068202e-05,
+      "loss": 1.5926,
+      "step": 4250
+    },
+    {
+      "epoch": 1.4970846749630145,
+      "grad_norm": 0.48151772986247815,
+      "learning_rate": 9.473093812668182e-05,
+      "loss": 1.5936,
+      "step": 4300
+    },
+    {
+      "epoch": 1.4970846749630145,
+      "eval_loss": 1.6123466491699219,
+      "eval_runtime": 14.0881,
+      "eval_samples_per_second": 70.982,
+      "eval_steps_per_second": 2.271,
+      "step": 4300
+    },
+    {
+      "epoch": 1.5144896005569577,
+      "grad_norm": 1.1271863223922003,
+      "learning_rate": 9.460851608370794e-05,
+      "loss": 1.6012,
+      "step": 4350
+    },
+    {
+      "epoch": 1.5318945261509007,
+      "grad_norm": 0.8558669669849335,
+      "learning_rate": 9.448476998663825e-05,
+      "loss": 1.605,
+      "step": 4400
+    },
+    {
+      "epoch": 1.5318945261509007,
+      "eval_loss": 1.6140981912612915,
+      "eval_runtime": 14.1256,
+      "eval_samples_per_second": 70.793,
+      "eval_steps_per_second": 2.265,
+      "step": 4400
+    },
+    {
+      "epoch": 1.5492994517448437,
+      "grad_norm": 0.7276127450869437,
+      "learning_rate": 9.435970355009615e-05,
+      "loss": 1.5938,
+      "step": 4450
+    },
+    {
+      "epoch": 1.566704377338787,
+      "grad_norm": 0.6065688198096086,
+      "learning_rate": 9.423332052833916e-05,
+      "loss": 1.5946,
+      "step": 4500
+    },
+    {
+      "epoch": 1.566704377338787,
+      "eval_loss": 1.611683964729309,
+      "eval_runtime": 14.0436,
+      "eval_samples_per_second": 71.207,
+      "eval_steps_per_second": 2.279,
+      "step": 4500
+    },
+    {
+      "epoch": 1.58410930293273,
+      "grad_norm": 0.7748024258482299,
+      "learning_rate": 9.410562471514616e-05,
+      "loss": 1.5894,
+      "step": 4550
+    },
+    {
+      "epoch": 1.601514228526673,
+      "grad_norm": 0.48917881847751543,
+      "learning_rate": 9.397661994370357e-05,
+      "loss": 1.5877,
+      "step": 4600
+    },
+    {
+      "epoch": 1.601514228526673,
+      "eval_loss": 1.6069624423980713,
+      "eval_runtime": 14.0735,
+      "eval_samples_per_second": 71.056,
+      "eval_steps_per_second": 2.274,
+      "step": 4600
+    },
+    {
+      "epoch": 1.6189191541206163,
+      "grad_norm": 0.8166564830453485,
+      "learning_rate": 9.384631008649027e-05,
+      "loss": 1.5875,
+      "step": 4650
+    },
+    {
+      "epoch": 1.636324079714559,
+      "grad_norm": 0.9485787011897893,
+      "learning_rate": 9.371469905516128e-05,
+      "loss": 1.5926,
+      "step": 4700
+    },
+    {
+      "epoch": 1.636324079714559,
+      "eval_loss": 1.6103551387786865,
+      "eval_runtime": 14.0489,
+      "eval_samples_per_second": 71.18,
+      "eval_steps_per_second": 2.278,
+      "step": 4700
+    },
+    {
+      "epoch": 1.6537290053085023,
+      "grad_norm": 0.6608190035209371,
+      "learning_rate": 9.358179080043047e-05,
+      "loss": 1.5852,
+      "step": 4750
+    },
+    {
+      "epoch": 1.6711339309024456,
+      "grad_norm": 0.5091041850584289,
+      "learning_rate": 9.344758931195186e-05,
+      "loss": 1.5818,
+      "step": 4800
+    },
+    {
+      "epoch": 1.6711339309024456,
+      "eval_loss": 1.6055699586868286,
+      "eval_runtime": 14.0386,
+      "eval_samples_per_second": 71.232,
+      "eval_steps_per_second": 2.279,
+      "step": 4800
+    },
+    {
+      "epoch": 1.6885388564963884,
+      "grad_norm": 0.4809752811498165,
+      "learning_rate": 9.331209861819991e-05,
+      "loss": 1.5945,
+      "step": 4850
+    },
+    {
+      "epoch": 1.7059437820903316,
+      "grad_norm": 1.16696044120828,
+      "learning_rate": 9.31753227863486e-05,
+      "loss": 1.5906,
+      "step": 4900
+    },
+    {
+      "epoch": 1.7059437820903316,
+      "eval_loss": 1.602495551109314,
+      "eval_runtime": 14.0638,
+      "eval_samples_per_second": 71.104,
+      "eval_steps_per_second": 2.275,
+      "step": 4900
+    },
+    {
+      "epoch": 1.7233487076842746,
+      "grad_norm": 0.7703478252526429,
+      "learning_rate": 9.303726592214927e-05,
+      "loss": 1.5759,
+      "step": 4950
+    },
+    {
+      "epoch": 1.7407536332782176,
+      "grad_norm": 0.4326591794595183,
+      "learning_rate": 9.289793216980748e-05,
+      "loss": 1.589,
+      "step": 5000
+    },
+    {
+      "epoch": 1.7407536332782176,
+      "eval_loss": 1.598211646080017,
+      "eval_runtime": 14.05,
+      "eval_samples_per_second": 71.174,
+      "eval_steps_per_second": 2.278,
+      "step": 5000
+    },
+    {
+      "epoch": 1.7581585588721609,
+      "grad_norm": 0.9150661442715593,
+      "learning_rate": 9.275732571185852e-05,
+      "loss": 1.5925,
+      "step": 5050
+    },
+    {
+      "epoch": 1.775563484466104,
+      "grad_norm": 0.4835138015080412,
+      "learning_rate": 9.261545076904189e-05,
+      "loss": 1.587,
+      "step": 5100
+    },
+    {
+      "epoch": 1.775563484466104,
+      "eval_loss": 1.5962464809417725,
+      "eval_runtime": 14.0435,
+      "eval_samples_per_second": 71.207,
+      "eval_steps_per_second": 2.279,
+      "step": 5100
+    },
+    {
+      "epoch": 1.792968410060047,
+      "grad_norm": 0.8246740616874354,
+      "learning_rate": 9.247231160017462e-05,
+      "loss": 1.5845,
+      "step": 5150
+    },
+    {
+      "epoch": 1.8103733356539902,
+      "grad_norm": 0.7636936218440887,
+      "learning_rate": 9.232791250202342e-05,
+      "loss": 1.5789,
+      "step": 5200
+    },
+    {
+      "epoch": 1.8103733356539902,
+      "eval_loss": 1.5962697267532349,
+      "eval_runtime": 14.0724,
+      "eval_samples_per_second": 71.061,
+      "eval_steps_per_second": 2.274,
+      "step": 5200
+    },
+    {
+      "epoch": 1.8277782612479332,
+      "grad_norm": 0.5278061111679693,
+      "learning_rate": 9.218225780917564e-05,
+      "loss": 1.5784,
+      "step": 5250
+    },
+    {
+      "epoch": 1.8451831868418762,
+      "grad_norm": 0.5521436007234811,
+      "learning_rate": 9.203535189390927e-05,
+      "loss": 1.5859,
+      "step": 5300
+    },
+    {
+      "epoch": 1.8451831868418762,
+      "eval_loss": 1.589383840560913,
+      "eval_runtime": 14.0972,
+      "eval_samples_per_second": 70.936,
+      "eval_steps_per_second": 2.27,
+      "step": 5300
+    },
+    {
+      "epoch": 1.8625881124358195,
+      "grad_norm": 0.9153838912238841,
+      "learning_rate": 9.188719916606157e-05,
+      "loss": 1.5767,
+      "step": 5350
+    },
+    {
+      "epoch": 1.8799930380297625,
+      "grad_norm": 0.5869179835862129,
+      "learning_rate": 9.17378040728968e-05,
+      "loss": 1.5771,
+      "step": 5400
+    },
+    {
+      "epoch": 1.8799930380297625,
+      "eval_loss": 1.5878838300704956,
+      "eval_runtime": 14.0586,
+      "eval_samples_per_second": 71.131,
+      "eval_steps_per_second": 2.276,
+      "step": 5400
+    },
+    {
+      "epoch": 1.8973979636237055,
+      "grad_norm": 0.8157168714834181,
+      "learning_rate": 9.158717109897263e-05,
+      "loss": 1.5626,
+      "step": 5450
+    },
+    {
+      "epoch": 1.9148028892176487,
+      "grad_norm": 0.7455391308200009,
+      "learning_rate": 9.14353047660056e-05,
+      "loss": 1.5651,
+      "step": 5500
+    },
+    {
+      "epoch": 1.9148028892176487,
+      "eval_loss": 1.5843595266342163,
+      "eval_runtime": 14.0141,
+      "eval_samples_per_second": 71.356,
+      "eval_steps_per_second": 2.283,
+      "step": 5500
+    },
+    {
+      "epoch": 1.9322078148115915,
+      "grad_norm": 0.48742202866618534,
+      "learning_rate": 9.128220963273532e-05,
+      "loss": 1.5806,
+      "step": 5550
+    },
+    {
+      "epoch": 1.9496127404055348,
+      "grad_norm": 0.49018002201797567,
+      "learning_rate": 9.112789029478769e-05,
+      "loss": 1.5715,
+      "step": 5600
+    },
+    {
+      "epoch": 1.9496127404055348,
+      "eval_loss": 1.583487868309021,
+      "eval_runtime": 14.076,
+      "eval_samples_per_second": 71.043,
+      "eval_steps_per_second": 2.273,
+      "step": 5600
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 28720,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 800,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5432367833415680.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50c0acabb981dfa7ab41f4379309e8fe958f9a7dc2fea40dc8e04a65181552bf
+size 7416