Upload folder using huggingface_hub

added_tokens.json

@@ -0,0 +1,13 @@
+{
+  "<|assistant|>": 32001,
+  "<|endoftext|>": 32000,
+  "<|end|>": 32007,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|system|>": 32006,
+  "<|user|>": 32010
+}

config.json

@@ -0,0 +1,153 @@
+{
+  "_name_or_path": "./checkpoint/llmga-Phi-3-mini-128k-full-finetune",
+  "architectures": [
+    "LlavaPhi3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_phi3.Phi3Config",
+    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
+  },
+  "bos_token_id": 1,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 32000,
+  "freeze_mm_mlp_adapter": false,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "image_aspect_ratio": "resizesquare",
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mm_hidden_size": 1024,
+  "mm_patch_merge_type": "flat",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "mm_vision_tower": "openai/clip-vit-large-patch14-336",
+  "model_type": "llava_Phi3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "original_max_position_embeddings": 4096,
+  "pad_token_id": 32000,
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "long_factor": [
+      1.0299999713897705,
+      1.0499999523162842,
+      1.0499999523162842,
+      1.0799999237060547,
+      1.2299998998641968,
+      1.2299998998641968,
+      1.2999999523162842,
+      1.4499999284744263,
+      1.5999999046325684,
+      1.6499998569488525,
+      1.8999998569488525,
+      2.859999895095825,
+      3.68999981880188,
+      5.419999599456787,
+      5.489999771118164,
+      5.489999771118164,
+      9.09000015258789,
+      11.579999923706055,
+      15.65999984741211,
+      15.769999504089355,
+      15.789999961853027,
+      18.360000610351562,
+      21.989999771118164,
+      23.079999923706055,
+      30.009998321533203,
+      32.35000228881836,
+      32.590003967285156,
+      35.56000518798828,
+      39.95000457763672,
+      53.840003967285156,
+      56.20000457763672,
+      57.95000457763672,
+      59.29000473022461,
+      59.77000427246094,
+      59.920005798339844,
+      61.190006256103516,
+      61.96000671386719,
+      62.50000762939453,
+      63.3700065612793,
+      63.48000717163086,
+      63.48000717163086,
+      63.66000747680664,
+      63.850006103515625,
+      64.08000946044922,
+      64.760009765625,
+      64.80001068115234,
+      64.81001281738281,
+      64.81001281738281
+    ],
+    "short_factor": [
+      1.05,
+      1.05,
+      1.05,
+      1.1,
+      1.1,
+      1.1500000000000001,
+      1.2000000000000002,
+      1.2500000000000002,
+      1.3000000000000003,
+      1.3500000000000003,
+      1.5000000000000004,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.0500000000000007,
+      2.0500000000000007,
+      2.0500000000000007,
+      2.1000000000000005,
+      2.1000000000000005,
+      2.1000000000000005,
+      2.1500000000000004,
+      2.1500000000000004,
+      2.3499999999999996,
+      2.549999999999999,
+      2.5999999999999988,
+      2.5999999999999988,
+      2.7499999999999982,
+      2.849999999999998,
+      2.849999999999998,
+      2.9499999999999975
+    ],
+    "type": "su"
+  },
+  "rope_theta": 10000.0,
+  "sliding_window": 262144,
+  "tie_word_embeddings": false,
+  "tokenizer_model_max_length": 2048,
+  "tokenizer_padding_side": "right",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.41.2",
+  "tune_mm_mlp_adapter": false,
+  "use_cache": true,
+  "use_mm_proj": true,
+  "vocab_size": 32064
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    32000,
+    32001,
+    32007
+  ],
+  "pad_token_id": 32000,
+  "transformers_version": "4.41.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fbfb6473fe41a68fee6070a686e8c93d0985eb8ae6ea3380d50ccb87a6a4abb
+size 4972489328

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62a3d25b743a45f01fab37115ac87bec8f3a4a7450ba7d2cf1f4ce442c92ddfb
+size 3301947368

model.safetensors.index.json

@@ -0,0 +1,597 @@
+{
+  "metadata": {
+    "total_size": 8274352128
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.mm_projector.0.bias": "model-00002-of-00002.safetensors",
+    "model.mm_projector.0.weight": "model-00002-of-00002.safetensors",
+    "model.mm_projector.2.bias": "model-00002-of-00002.safetensors",
+    "model.mm_projector.2.weight": "model-00002-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.embeddings.class_embedding": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.embeddings.position_embedding.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.post_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.post_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.pre_layrnorm.bias": "model-00002-of-00002.safetensors",
+    "model.vision_tower.vision_tower.vision_model.pre_layrnorm.weight": "model-00002-of-00002.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,132 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "32000": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|placeholder1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|placeholder2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|placeholder3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<|placeholder4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<|placeholder5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<|placeholder6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "legacy": false,
+  "model_max_length": 2048,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

trainer_state.json

@@ -0,0 +1,2002 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 280,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0035714285714285713,
+      "grad_norm": 1.5094072773241762,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 1.0151,
+      "step": 1
+    },
+    {
+      "epoch": 0.007142857142857143,
+      "grad_norm": 1.2261567161033138,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 0.9601,
+      "step": 2
+    },
+    {
+      "epoch": 0.010714285714285714,
+      "grad_norm": 1.3789264302597328,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 0.9805,
+      "step": 3
+    },
+    {
+      "epoch": 0.014285714285714285,
+      "grad_norm": 1.1735489322854569,
+      "learning_rate": 8.888888888888888e-06,
+      "loss": 0.9329,
+      "step": 4
+    },
+    {
+      "epoch": 0.017857142857142856,
+      "grad_norm": 0.9728686614590386,
+      "learning_rate": 1.1111111111111113e-05,
+      "loss": 0.9093,
+      "step": 5
+    },
+    {
+      "epoch": 0.02142857142857143,
+      "grad_norm": 0.8337322061485447,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 0.8627,
+      "step": 6
+    },
+    {
+      "epoch": 0.025,
+      "grad_norm": 0.8077678473828049,
+      "learning_rate": 1.555555555555556e-05,
+      "loss": 0.8997,
+      "step": 7
+    },
+    {
+      "epoch": 0.02857142857142857,
+      "grad_norm": 1.1544947424109089,
+      "learning_rate": 1.7777777777777777e-05,
+      "loss": 0.8306,
+      "step": 8
+    },
+    {
+      "epoch": 0.03214285714285714,
+      "grad_norm": 0.9365331968567063,
+      "learning_rate": 2e-05,
+      "loss": 0.8259,
+      "step": 9
+    },
+    {
+      "epoch": 0.03571428571428571,
+      "grad_norm": 0.8733703386322631,
+      "learning_rate": 1.9999328066483867e-05,
+      "loss": 0.7634,
+      "step": 10
+    },
+    {
+      "epoch": 0.039285714285714285,
+      "grad_norm": 1.0508833886855795,
+      "learning_rate": 1.9997312356234385e-05,
+      "loss": 0.7844,
+      "step": 11
+    },
+    {
+      "epoch": 0.04285714285714286,
+      "grad_norm": 0.8733780761590844,
+      "learning_rate": 1.999395314013622e-05,
+      "loss": 0.7007,
+      "step": 12
+    },
+    {
+      "epoch": 0.04642857142857143,
+      "grad_norm": 0.8491820034489115,
+      "learning_rate": 1.998925086962334e-05,
+      "loss": 0.6727,
+      "step": 13
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 0.9878586965754647,
+      "learning_rate": 1.998320617661839e-05,
+      "loss": 0.6525,
+      "step": 14
+    },
+    {
+      "epoch": 0.05357142857142857,
+      "grad_norm": 0.8118877765114055,
+      "learning_rate": 1.997581987344772e-05,
+      "loss": 0.6359,
+      "step": 15
+    },
+    {
+      "epoch": 0.05714285714285714,
+      "grad_norm": 0.7086914890601413,
+      "learning_rate": 1.9967092952732266e-05,
+      "loss": 0.585,
+      "step": 16
+    },
+    {
+      "epoch": 0.060714285714285714,
+      "grad_norm": 0.8192484120355384,
+      "learning_rate": 1.9957026587254136e-05,
+      "loss": 0.6104,
+      "step": 17
+    },
+    {
+      "epoch": 0.06428571428571428,
+      "grad_norm": 0.7161431881584631,
+      "learning_rate": 1.9945622129799e-05,
+      "loss": 0.5429,
+      "step": 18
+    },
+    {
+      "epoch": 0.06785714285714285,
+      "grad_norm": 0.7007323929826601,
+      "learning_rate": 1.9932881112974298e-05,
+      "loss": 0.5499,
+      "step": 19
+    },
+    {
+      "epoch": 0.07142857142857142,
+      "grad_norm": 0.6115671214910369,
+      "learning_rate": 1.9918805249003272e-05,
+      "loss": 0.474,
+      "step": 20
+    },
+    {
+      "epoch": 0.075,
+      "grad_norm": 0.6578343481302569,
+      "learning_rate": 1.9903396429494882e-05,
+      "loss": 0.4351,
+      "step": 21
+    },
+    {
+      "epoch": 0.07857142857142857,
+      "grad_norm": 0.5724846059419685,
+      "learning_rate": 1.9886656725189575e-05,
+      "loss": 0.4171,
+      "step": 22
+    },
+    {
+      "epoch": 0.08214285714285714,
+      "grad_norm": 0.599137384328337,
+      "learning_rate": 1.9868588385681035e-05,
+      "loss": 0.3886,
+      "step": 23
+    },
+    {
+      "epoch": 0.08571428571428572,
+      "grad_norm": 0.5215963930855563,
+      "learning_rate": 1.9849193839113833e-05,
+      "loss": 0.3251,
+      "step": 24
+    },
+    {
+      "epoch": 0.08928571428571429,
+      "grad_norm": 0.5047278640358218,
+      "learning_rate": 1.9828475691857148e-05,
+      "loss": 0.3477,
+      "step": 25
+    },
+    {
+      "epoch": 0.09285714285714286,
+      "grad_norm": 0.8532208856128077,
+      "learning_rate": 1.9806436728154484e-05,
+      "loss": 0.3739,
+      "step": 26
+    },
+    {
+      "epoch": 0.09642857142857143,
+      "grad_norm": 0.546355655031926,
+      "learning_rate": 1.9783079909749516e-05,
+      "loss": 0.3526,
+      "step": 27
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.503619493558213,
+      "learning_rate": 1.975840837548807e-05,
+      "loss": 0.302,
+      "step": 28
+    },
+    {
+      "epoch": 0.10357142857142858,
+      "grad_norm": 0.6026201410093914,
+      "learning_rate": 1.9732425440896298e-05,
+      "loss": 0.4273,
+      "step": 29
+    },
+    {
+      "epoch": 0.10714285714285714,
+      "grad_norm": 0.5114090959113702,
+      "learning_rate": 1.9705134597735113e-05,
+      "loss": 0.3331,
+      "step": 30
+    },
+    {
+      "epoch": 0.11071428571428571,
+      "grad_norm": 0.42050099872679103,
+      "learning_rate": 1.9676539513530967e-05,
+      "loss": 0.2354,
+      "step": 31
+    },
+    {
+      "epoch": 0.11428571428571428,
+      "grad_norm": 0.4478105656271568,
+      "learning_rate": 1.9646644031082948e-05,
+      "loss": 0.2833,
+      "step": 32
+    },
+    {
+      "epoch": 0.11785714285714285,
+      "grad_norm": 0.5071836938044539,
+      "learning_rate": 1.9615452167946383e-05,
+      "loss": 0.3113,
+      "step": 33
+    },
+    {
+      "epoch": 0.12142857142857143,
+      "grad_norm": 0.5175472912935297,
+      "learning_rate": 1.958296811589293e-05,
+      "loss": 0.2869,
+      "step": 34
+    },
+    {
+      "epoch": 0.125,
+      "grad_norm": 0.5691685554208916,
+      "learning_rate": 1.954919624034725e-05,
+      "loss": 0.3124,
+      "step": 35
+    },
+    {
+      "epoch": 0.12857142857142856,
+      "grad_norm": 0.5033724920902282,
+      "learning_rate": 1.951414107980036e-05,
+      "loss": 0.2837,
+      "step": 36
+    },
+    {
+      "epoch": 0.13214285714285715,
+      "grad_norm": 0.5132265455132116,
+      "learning_rate": 1.9477807345199717e-05,
+      "loss": 0.2672,
+      "step": 37
+    },
+    {
+      "epoch": 0.1357142857142857,
+      "grad_norm": 0.45135896832593086,
+      "learning_rate": 1.9440199919316125e-05,
+      "loss": 0.2625,
+      "step": 38
+    },
+    {
+      "epoch": 0.1392857142857143,
+      "grad_norm": 0.5770043417852031,
+      "learning_rate": 1.9401323856087573e-05,
+      "loss": 0.2564,
+      "step": 39
+    },
+    {
+      "epoch": 0.14285714285714285,
+      "grad_norm": 0.4833137787136883,
+      "learning_rate": 1.936118437994003e-05,
+      "loss": 0.2949,
+      "step": 40
+    },
+    {
+      "epoch": 0.14642857142857144,
+      "grad_norm": 0.47925753151438966,
+      "learning_rate": 1.9319786885085366e-05,
+      "loss": 0.2749,
+      "step": 41
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 0.48378617286710246,
+      "learning_rate": 1.927713693479643e-05,
+      "loss": 0.2758,
+      "step": 42
+    },
+    {
+      "epoch": 0.15357142857142858,
+      "grad_norm": 0.45827404889673506,
+      "learning_rate": 1.923324026065944e-05,
+      "loss": 0.2551,
+      "step": 43
+    },
+    {
+      "epoch": 0.15714285714285714,
+      "grad_norm": 0.4293728892267699,
+      "learning_rate": 1.918810276180372e-05,
+      "loss": 0.2226,
+      "step": 44
+    },
+    {
+      "epoch": 0.16071428571428573,
+      "grad_norm": 0.5046863943049192,
+      "learning_rate": 1.9141730504108923e-05,
+      "loss": 0.2302,
+      "step": 45
+    },
+    {
+      "epoch": 0.16428571428571428,
+      "grad_norm": 0.4476317734558537,
+      "learning_rate": 1.9094129719389886e-05,
+      "loss": 0.2422,
+      "step": 46
+    },
+    {
+      "epoch": 0.16785714285714284,
+      "grad_norm": 0.46562703123431065,
+      "learning_rate": 1.904530680455914e-05,
+      "loss": 0.2592,
+      "step": 47
+    },
+    {
+      "epoch": 0.17142857142857143,
+      "grad_norm": 0.43987974040881184,
+      "learning_rate": 1.8995268320767254e-05,
+      "loss": 0.2331,
+      "step": 48
+    },
+    {
+      "epoch": 0.175,
+      "grad_norm": 0.4189071255346078,
+      "learning_rate": 1.894402099252109e-05,
+      "loss": 0.2138,
+      "step": 49
+    },
+    {
+      "epoch": 0.17857142857142858,
+      "grad_norm": 0.5022882831214411,
+      "learning_rate": 1.889157170678015e-05,
+      "loss": 0.2276,
+      "step": 50
+    },
+    {
+      "epoch": 0.18214285714285713,
+      "grad_norm": 0.4242382698400041,
+      "learning_rate": 1.8837927512031022e-05,
+      "loss": 0.1954,
+      "step": 51
+    },
+    {
+      "epoch": 0.18571428571428572,
+      "grad_norm": 0.46729091179001897,
+      "learning_rate": 1.8783095617340193e-05,
+      "loss": 0.2123,
+      "step": 52
+    },
+    {
+      "epoch": 0.18928571428571428,
+      "grad_norm": 0.34912389539750716,
+      "learning_rate": 1.872708339138522e-05,
+      "loss": 0.1539,
+      "step": 53
+    },
+    {
+      "epoch": 0.19285714285714287,
+      "grad_norm": 0.34193140502921543,
+      "learning_rate": 1.866989836146449e-05,
+      "loss": 0.1567,
+      "step": 54
+    },
+    {
+      "epoch": 0.19642857142857142,
+      "grad_norm": 0.43454794382330175,
+      "learning_rate": 1.861154821248565e-05,
+      "loss": 0.2267,
+      "step": 55
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 0.42705246479088504,
+      "learning_rate": 1.8552040785932846e-05,
+      "loss": 0.188,
+      "step": 56
+    },
+    {
+      "epoch": 0.20357142857142857,
+      "grad_norm": 0.4006780695936107,
+      "learning_rate": 1.849138407881296e-05,
+      "loss": 0.1789,
+      "step": 57
+    },
+    {
+      "epoch": 0.20714285714285716,
+      "grad_norm": 0.42480324904794664,
+      "learning_rate": 1.8429586242580884e-05,
+      "loss": 0.2062,
+      "step": 58
+    },
+    {
+      "epoch": 0.21071428571428572,
+      "grad_norm": 0.4304497167935713,
+      "learning_rate": 1.8366655582044096e-05,
+      "loss": 0.1521,
+      "step": 59
+    },
+    {
+      "epoch": 0.21428571428571427,
+      "grad_norm": 0.5269250205907539,
+      "learning_rate": 1.83026005542466e-05,
+      "loss": 0.2221,
+      "step": 60
+    },
+    {
+      "epoch": 0.21785714285714286,
+      "grad_norm": 0.43416177786575055,
+      "learning_rate": 1.8237429767332407e-05,
+      "loss": 0.1925,
+      "step": 61
+    },
+    {
+      "epoch": 0.22142857142857142,
+      "grad_norm": 0.33226365056585994,
+      "learning_rate": 1.8171151979388715e-05,
+      "loss": 0.1602,
+      "step": 62
+    },
+    {
+      "epoch": 0.225,
+      "grad_norm": 0.42522622708750485,
+      "learning_rate": 1.8103776097268942e-05,
+      "loss": 0.2303,
+      "step": 63
+    },
+    {
+      "epoch": 0.22857142857142856,
+      "grad_norm": 0.31629924219736033,
+      "learning_rate": 1.803531117539577e-05,
+      "loss": 0.1675,
+      "step": 64
+    },
+    {
+      "epoch": 0.23214285714285715,
+      "grad_norm": 0.33553863337144985,
+      "learning_rate": 1.7965766414544328e-05,
+      "loss": 0.1454,
+      "step": 65
+    },
+    {
+      "epoch": 0.2357142857142857,
+      "grad_norm": 0.39694233093962233,
+      "learning_rate": 1.7895151160605758e-05,
+      "loss": 0.1651,
+      "step": 66
+    },
+    {
+      "epoch": 0.2392857142857143,
+      "grad_norm": 0.3909301813501153,
+      "learning_rate": 1.782347490333123e-05,
+      "loss": 0.1836,
+      "step": 67
+    },
+    {
+      "epoch": 0.24285714285714285,
+      "grad_norm": 0.3713849341683806,
+      "learning_rate": 1.775074727505667e-05,
+      "loss": 0.1789,
+      "step": 68
+    },
+    {
+      "epoch": 0.24642857142857144,
+      "grad_norm": 0.33263049248548554,
+      "learning_rate": 1.7676978049408262e-05,
+      "loss": 0.1535,
+      "step": 69
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 0.4039965531248106,
+      "learning_rate": 1.7602177139989046e-05,
+      "loss": 0.1619,
+      "step": 70
+    },
+    {
+      "epoch": 0.25357142857142856,
+      "grad_norm": 0.4449361845478868,
+      "learning_rate": 1.7526354599046637e-05,
+      "loss": 0.1982,
+      "step": 71
+    },
+    {
+      "epoch": 0.2571428571428571,
+      "grad_norm": 0.30951049443853174,
+      "learning_rate": 1.7449520616122344e-05,
+      "loss": 0.1608,
+      "step": 72
+    },
+    {
+      "epoch": 0.26071428571428573,
+      "grad_norm": 0.30303961161237486,
+      "learning_rate": 1.7371685516681825e-05,
+      "loss": 0.1452,
+      "step": 73
+    },
+    {
+      "epoch": 0.2642857142857143,
+      "grad_norm": 0.37534495021092973,
+      "learning_rate": 1.7292859760727493e-05,
+      "loss": 0.1939,
+      "step": 74
+    },
+    {
+      "epoch": 0.26785714285714285,
+      "grad_norm": 0.29341418722705936,
+      "learning_rate": 1.721305394139282e-05,
+      "loss": 0.1235,
+      "step": 75
+    },
+    {
+      "epoch": 0.2714285714285714,
+      "grad_norm": 0.27507736051060533,
+      "learning_rate": 1.7132278783518756e-05,
+      "loss": 0.1341,
+      "step": 76
+    },
+    {
+      "epoch": 0.275,
+      "grad_norm": 0.32989837216671697,
+      "learning_rate": 1.7050545142212483e-05,
+      "loss": 0.1892,
+      "step": 77
+    },
+    {
+      "epoch": 0.2785714285714286,
+      "grad_norm": 0.3451592284082488,
+      "learning_rate": 1.696786400138859e-05,
+      "loss": 0.1617,
+      "step": 78
+    },
+    {
+      "epoch": 0.28214285714285714,
+      "grad_norm": 0.40263328568666046,
+      "learning_rate": 1.6884246472293018e-05,
+      "loss": 0.2107,
+      "step": 79
+    },
+    {
+      "epoch": 0.2857142857142857,
+      "grad_norm": 0.3229086446687719,
+      "learning_rate": 1.679970379200983e-05,
+      "loss": 0.1752,
+      "step": 80
+    },
+    {
+      "epoch": 0.2892857142857143,
+      "grad_norm": 0.3006007923046572,
+      "learning_rate": 1.6714247321951106e-05,
+      "loss": 0.1412,
+      "step": 81
+    },
+    {
+      "epoch": 0.29285714285714287,
+      "grad_norm": 0.37122613594080844,
+      "learning_rate": 1.6627888546330136e-05,
+      "loss": 0.2006,
+      "step": 82
+    },
+    {
+      "epoch": 0.29642857142857143,
+      "grad_norm": 0.3793991954120453,
+      "learning_rate": 1.654063907061807e-05,
+      "loss": 0.1817,
+      "step": 83
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 0.3049450256392541,
+      "learning_rate": 1.64525106199843e-05,
+      "loss": 0.1645,
+      "step": 84
+    },
+    {
+      "epoch": 0.30357142857142855,
+      "grad_norm": 0.27500204317093907,
+      "learning_rate": 1.6363515037720774e-05,
+      "loss": 0.1615,
+      "step": 85
+    },
+    {
+      "epoch": 0.30714285714285716,
+      "grad_norm": 0.28823525091179314,
+      "learning_rate": 1.6273664283650393e-05,
+      "loss": 0.182,
+      "step": 86
+    },
+    {
+      "epoch": 0.3107142857142857,
+      "grad_norm": 0.2505654102121107,
+      "learning_rate": 1.6182970432519772e-05,
+      "loss": 0.1176,
+      "step": 87
+    },
+    {
+      "epoch": 0.3142857142857143,
+      "grad_norm": 0.30588157473716326,
+      "learning_rate": 1.609144567237658e-05,
+      "loss": 0.1701,
+      "step": 88
+    },
+    {
+      "epoch": 0.31785714285714284,
+      "grad_norm": 0.3337306834899281,
+      "learning_rate": 1.5999102302931585e-05,
+      "loss": 0.1445,
+      "step": 89
+    },
+    {
+      "epoch": 0.32142857142857145,
+      "grad_norm": 0.4048264056659017,
+      "learning_rate": 1.5905952733905777e-05,
+      "loss": 0.2287,
+      "step": 90
+    },
+    {
+      "epoch": 0.325,
+      "grad_norm": 0.3486455924270133,
+      "learning_rate": 1.5812009483362643e-05,
+      "loss": 0.1997,
+      "step": 91
+    },
+    {
+      "epoch": 0.32857142857142857,
+      "grad_norm": 0.4019411694484333,
+      "learning_rate": 1.5717285176025913e-05,
+      "loss": 0.1611,
+      "step": 92
+    },
+    {
+      "epoch": 0.33214285714285713,
+      "grad_norm": 0.4461157561097472,
+      "learning_rate": 1.5621792541582968e-05,
+      "loss": 0.2028,
+      "step": 93
+    },
+    {
+      "epoch": 0.3357142857142857,
+      "grad_norm": 0.26983028718605406,
+      "learning_rate": 1.552554441297413e-05,
+      "loss": 0.1739,
+      "step": 94
+    },
+    {
+      "epoch": 0.3392857142857143,
+      "grad_norm": 0.3682824119957896,
+      "learning_rate": 1.5428553724668103e-05,
+      "loss": 0.1875,
+      "step": 95
+    },
+    {
+      "epoch": 0.34285714285714286,
+      "grad_norm": 0.27852461917505517,
+      "learning_rate": 1.533083351092372e-05,
+      "loss": 0.1211,
+      "step": 96
+    },
+    {
+      "epoch": 0.3464285714285714,
+      "grad_norm": 0.2466615958620265,
+      "learning_rate": 1.5232396904038352e-05,
+      "loss": 0.1148,
+      "step": 97
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 0.27490316964093686,
+      "learning_rate": 1.5133257132583074e-05,
+      "loss": 0.1492,
+      "step": 98
+    },
+    {
+      "epoch": 0.3535714285714286,
+      "grad_norm": 0.36693430023627566,
+      "learning_rate": 1.503342751962493e-05,
+      "loss": 0.1966,
+      "step": 99
+    },
+    {
+      "epoch": 0.35714285714285715,
+      "grad_norm": 0.3038332005221938,
+      "learning_rate": 1.4932921480936491e-05,
+      "loss": 0.1722,
+      "step": 100
+    },
+    {
+      "epoch": 0.3607142857142857,
+      "grad_norm": 0.3188144302422291,
+      "learning_rate": 1.4831752523192949e-05,
+      "loss": 0.1737,
+      "step": 101
+    },
+    {
+      "epoch": 0.36428571428571427,
+      "grad_norm": 0.3547830918183239,
+      "learning_rate": 1.4729934242157005e-05,
+      "loss": 0.1942,
+      "step": 102
+    },
+    {
+      "epoch": 0.3678571428571429,
+      "grad_norm": 0.35280055034073526,
+      "learning_rate": 1.4627480320851775e-05,
+      "loss": 0.2125,
+      "step": 103
+    },
+    {
+      "epoch": 0.37142857142857144,
+      "grad_norm": 0.3992974205788511,
+      "learning_rate": 1.4524404527721977e-05,
+      "loss": 0.1822,
+      "step": 104
+    },
+    {
+      "epoch": 0.375,
+      "grad_norm": 0.2822925801720912,
+      "learning_rate": 1.4420720714783635e-05,
+      "loss": 0.1982,
+      "step": 105
+    },
+    {
+      "epoch": 0.37857142857142856,
+      "grad_norm": 0.3344602401618272,
+      "learning_rate": 1.4316442815762543e-05,
+      "loss": 0.1998,
+      "step": 106
+    },
+    {
+      "epoch": 0.3821428571428571,
+      "grad_norm": 0.2880117007092726,
+      "learning_rate": 1.4211584844221771e-05,
+      "loss": 0.126,
+      "step": 107
+    },
+    {
+      "epoch": 0.38571428571428573,
+      "grad_norm": 0.29965781933906427,
+      "learning_rate": 1.4106160891678422e-05,
+      "loss": 0.1329,
+      "step": 108
+    },
+    {
+      "epoch": 0.3892857142857143,
+      "grad_norm": 0.23743122231056354,
+      "learning_rate": 1.4000185125709919e-05,
+      "loss": 0.1294,
+      "step": 109
+    },
+    {
+      "epoch": 0.39285714285714285,
+      "grad_norm": 0.3248974825475453,
+      "learning_rate": 1.3893671788050073e-05,
+      "loss": 0.1336,
+      "step": 110
+    },
+    {
+      "epoch": 0.3964285714285714,
+      "grad_norm": 0.22306284289830697,
+      "learning_rate": 1.3786635192675184e-05,
+      "loss": 0.1415,
+      "step": 111
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.41951553591564245,
+      "learning_rate": 1.3679089723880427e-05,
+      "loss": 0.1748,
+      "step": 112
+    },
+    {
+      "epoch": 0.4035714285714286,
+      "grad_norm": 0.21840731647775014,
+      "learning_rate": 1.35710498343468e-05,
+      "loss": 0.1046,
+      "step": 113
+    },
+    {
+      "epoch": 0.40714285714285714,
+      "grad_norm": 0.38462679391341553,
+      "learning_rate": 1.3462530043198874e-05,
+      "loss": 0.1973,
+      "step": 114
+    },
+    {
+      "epoch": 0.4107142857142857,
+      "grad_norm": 0.26934313162508683,
+      "learning_rate": 1.3353544934053618e-05,
+      "loss": 0.1375,
+      "step": 115
+    },
+    {
+      "epoch": 0.4142857142857143,
+      "grad_norm": 0.2544477777748366,
+      "learning_rate": 1.324410915306055e-05,
+      "loss": 0.1425,
+      "step": 116
+    },
+    {
+      "epoch": 0.41785714285714287,
+      "grad_norm": 0.24558119287897373,
+      "learning_rate": 1.3134237406933493e-05,
+      "loss": 0.1041,
+      "step": 117
+    },
+    {
+      "epoch": 0.42142857142857143,
+      "grad_norm": 0.3045031243515618,
+      "learning_rate": 1.3023944460974183e-05,
+      "loss": 0.1648,
+      "step": 118
+    },
+    {
+      "epoch": 0.425,
+      "grad_norm": 0.3407203340245133,
+      "learning_rate": 1.2913245137088024e-05,
+      "loss": 0.1792,
+      "step": 119
+    },
+    {
+      "epoch": 0.42857142857142855,
+      "grad_norm": 0.2099211870074925,
+      "learning_rate": 1.2802154311792196e-05,
+      "loss": 0.1095,
+      "step": 120
+    },
+    {
+      "epoch": 0.43214285714285716,
+      "grad_norm": 0.2598183580508858,
+      "learning_rate": 1.2690686914216475e-05,
+      "loss": 0.1397,
+      "step": 121
+    },
+    {
+      "epoch": 0.4357142857142857,
+      "grad_norm": 0.2235506753897087,
+      "learning_rate": 1.2578857924096935e-05,
+      "loss": 0.1143,
+      "step": 122
+    },
+    {
+      "epoch": 0.4392857142857143,
+      "grad_norm": 0.2543616472706576,
+      "learning_rate": 1.2466682369762883e-05,
+      "loss": 0.1272,
+      "step": 123
+    },
+    {
+      "epoch": 0.44285714285714284,
+      "grad_norm": 0.4011226680538365,
+      "learning_rate": 1.2354175326117252e-05,
+      "loss": 0.1983,
+      "step": 124
+    },
+    {
+      "epoch": 0.44642857142857145,
+      "grad_norm": 0.3493406011133788,
+      "learning_rate": 1.2241351912610726e-05,
+      "loss": 0.2002,
+      "step": 125
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 0.24215423055262641,
+      "learning_rate": 1.212822729120989e-05,
+      "loss": 0.1229,
+      "step": 126
+    },
+    {
+      "epoch": 0.45357142857142857,
+      "grad_norm": 0.3521084697628892,
+      "learning_rate": 1.2014816664359671e-05,
+      "loss": 0.1526,
+      "step": 127
+    },
+    {
+      "epoch": 0.45714285714285713,
+      "grad_norm": 0.31964058859578576,
+      "learning_rate": 1.190113527294032e-05,
+      "loss": 0.1522,
+      "step": 128
+    },
+    {
+      "epoch": 0.4607142857142857,
+      "grad_norm": 0.2460798969556437,
+      "learning_rate": 1.178719839421925e-05,
+      "loss": 0.1229,
+      "step": 129
+    },
+    {
+      "epoch": 0.4642857142857143,
+      "grad_norm": 0.2976745416013965,
+      "learning_rate": 1.1673021339797967e-05,
+      "loss": 0.1574,
+      "step": 130
+    },
+    {
+      "epoch": 0.46785714285714286,
+      "grad_norm": 0.27597995377817075,
+      "learning_rate": 1.15586194535544e-05,
+      "loss": 0.1312,
+      "step": 131
+    },
+    {
+      "epoch": 0.4714285714285714,
+      "grad_norm": 0.20695554030623772,
+      "learning_rate": 1.1444008109580884e-05,
+      "loss": 0.0914,
+      "step": 132
+    },
+    {
+      "epoch": 0.475,
+      "grad_norm": 0.30893043082120447,
+      "learning_rate": 1.1329202710118088e-05,
+      "loss": 0.1679,
+      "step": 133
+    },
+    {
+      "epoch": 0.4785714285714286,
+      "grad_norm": 0.33056398836471207,
+      "learning_rate": 1.1214218683485159e-05,
+      "loss": 0.1844,
+      "step": 134
+    },
+    {
+      "epoch": 0.48214285714285715,
+      "grad_norm": 0.2623319459839479,
+      "learning_rate": 1.1099071482006361e-05,
+      "loss": 0.1417,
+      "step": 135
+    },
+    {
+      "epoch": 0.4857142857142857,
+      "grad_norm": 0.3081145360553918,
+      "learning_rate": 1.0983776579934483e-05,
+      "loss": 0.1604,
+      "step": 136
+    },
+    {
+      "epoch": 0.48928571428571427,
+      "grad_norm": 0.4003584770029758,
+      "learning_rate": 1.0868349471371316e-05,
+      "loss": 0.2382,
+      "step": 137
+    },
+    {
+      "epoch": 0.4928571428571429,
+      "grad_norm": 0.23595184021475982,
+      "learning_rate": 1.0752805668185442e-05,
+      "loss": 0.1128,
+      "step": 138
+    },
+    {
+      "epoch": 0.49642857142857144,
+      "grad_norm": 0.3426077155449424,
+      "learning_rate": 1.0637160697927651e-05,
+      "loss": 0.1832,
+      "step": 139
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.2790332076283996,
+      "learning_rate": 1.0521430101744238e-05,
+      "loss": 0.149,
+      "step": 140
+    },
+    {
+      "epoch": 0.5035714285714286,
+      "grad_norm": 0.2794114725059522,
+      "learning_rate": 1.040562943228849e-05,
+      "loss": 0.1511,
+      "step": 141
+    },
+    {
+      "epoch": 0.5071428571428571,
+      "grad_norm": 0.21843838839618104,
+      "learning_rate": 1.0289774251630602e-05,
+      "loss": 0.1341,
+      "step": 142
+    },
+    {
+      "epoch": 0.5107142857142857,
+      "grad_norm": 0.24181152567288938,
+      "learning_rate": 1.0173880129166358e-05,
+      "loss": 0.1475,
+      "step": 143
+    },
+    {
+      "epoch": 0.5142857142857142,
+      "grad_norm": 0.22316879395770384,
+      "learning_rate": 1.0057962639524799e-05,
+      "loss": 0.1206,
+      "step": 144
+    },
+    {
+      "epoch": 0.5178571428571429,
+      "grad_norm": 0.28287874107637956,
+      "learning_rate": 9.942037360475205e-06,
+      "loss": 0.1295,
+      "step": 145
+    },
+    {
+      "epoch": 0.5214285714285715,
+      "grad_norm": 0.30431919248377765,
+      "learning_rate": 9.826119870833644e-06,
+      "loss": 0.1642,
+      "step": 146
+    },
+    {
+      "epoch": 0.525,
+      "grad_norm": 0.2937134104338438,
+      "learning_rate": 9.710225748369402e-06,
+      "loss": 0.1557,
+      "step": 147
+    },
+    {
+      "epoch": 0.5285714285714286,
+      "grad_norm": 0.1786321181599842,
+      "learning_rate": 9.594370567711512e-06,
+      "loss": 0.0936,
+      "step": 148
+    },
+    {
+      "epoch": 0.5321428571428571,
+      "grad_norm": 0.3018045804554168,
+      "learning_rate": 9.478569898255765e-06,
+      "loss": 0.1664,
+      "step": 149
+    },
+    {
+      "epoch": 0.5357142857142857,
+      "grad_norm": 0.2157758288495335,
+      "learning_rate": 9.362839302072354e-06,
+      "loss": 0.1024,
+      "step": 150
+    },
+    {
+      "epoch": 0.5392857142857143,
+      "grad_norm": 0.3516430379199773,
+      "learning_rate": 9.247194331814561e-06,
+      "loss": 0.1798,
+      "step": 151
+    },
+    {
+      "epoch": 0.5428571428571428,
+      "grad_norm": 0.24724631208071982,
+      "learning_rate": 9.131650528628688e-06,
+      "loss": 0.1507,
+      "step": 152
+    },
+    {
+      "epoch": 0.5464285714285714,
+      "grad_norm": 0.2652492386455145,
+      "learning_rate": 9.016223420065519e-06,
+      "loss": 0.1642,
+      "step": 153
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 0.259065432983257,
+      "learning_rate": 8.900928517993644e-06,
+      "loss": 0.1403,
+      "step": 154
+    },
+    {
+      "epoch": 0.5535714285714286,
+      "grad_norm": 0.258696120006156,
+      "learning_rate": 8.785781316514841e-06,
+      "loss": 0.1626,
+      "step": 155
+    },
+    {
+      "epoch": 0.5571428571428572,
+      "grad_norm": 0.26559972426937806,
+      "learning_rate": 8.670797289881915e-06,
+      "loss": 0.1311,
+      "step": 156
+    },
+    {
+      "epoch": 0.5607142857142857,
+      "grad_norm": 0.24255680373144262,
+      "learning_rate": 8.555991890419116e-06,
+      "loss": 0.1385,
+      "step": 157
+    },
+    {
+      "epoch": 0.5642857142857143,
+      "grad_norm": 0.2187837567432758,
+      "learning_rate": 8.441380546445603e-06,
+      "loss": 0.1007,
+      "step": 158
+    },
+    {
+      "epoch": 0.5678571428571428,
+      "grad_norm": 0.3754007934241826,
+      "learning_rate": 8.326978660202034e-06,
+      "loss": 0.2339,
+      "step": 159
+    },
+    {
+      "epoch": 0.5714285714285714,
+      "grad_norm": 0.291493903881618,
+      "learning_rate": 8.212801605780754e-06,
+      "loss": 0.1578,
+      "step": 160
+    },
+    {
+      "epoch": 0.575,
+      "grad_norm": 0.20017896968794144,
+      "learning_rate": 8.098864727059685e-06,
+      "loss": 0.0999,
+      "step": 161
+    },
+    {
+      "epoch": 0.5785714285714286,
+      "grad_norm": 0.2940625521302557,
+      "learning_rate": 7.985183335640332e-06,
+      "loss": 0.1688,
+      "step": 162
+    },
+    {
+      "epoch": 0.5821428571428572,
+      "grad_norm": 0.18618459726444583,
+      "learning_rate": 7.871772708790114e-06,
+      "loss": 0.0922,
+      "step": 163
+    },
+    {
+      "epoch": 0.5857142857142857,
+      "grad_norm": 0.2604400312724968,
+      "learning_rate": 7.758648087389277e-06,
+      "loss": 0.1366,
+      "step": 164
+    },
+    {
+      "epoch": 0.5892857142857143,
+      "grad_norm": 0.29864320094288466,
+      "learning_rate": 7.64582467388275e-06,
+      "loss": 0.175,
+      "step": 165
+    },
+    {
+      "epoch": 0.5928571428571429,
+      "grad_norm": 0.21688603331716927,
+      "learning_rate": 7.533317630237117e-06,
+      "loss": 0.1421,
+      "step": 166
+    },
+    {
+      "epoch": 0.5964285714285714,
+      "grad_norm": 0.3104896738837084,
+      "learning_rate": 7.421142075903067e-06,
+      "loss": 0.1601,
+      "step": 167
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.25537176187440613,
+      "learning_rate": 7.3093130857835245e-06,
+      "loss": 0.1307,
+      "step": 168
+    },
+    {
+      "epoch": 0.6035714285714285,
+      "grad_norm": 0.30448029888397105,
+      "learning_rate": 7.197845688207805e-06,
+      "loss": 0.1692,
+      "step": 169
+    },
+    {
+      "epoch": 0.6071428571428571,
+      "grad_norm": 0.2657221601971254,
+      "learning_rate": 7.086754862911982e-06,
+      "loss": 0.1575,
+      "step": 170
+    },
+    {
+      "epoch": 0.6107142857142858,
+      "grad_norm": 0.3242863420361999,
+      "learning_rate": 6.976055539025819e-06,
+      "loss": 0.1981,
+      "step": 171
+    },
+    {
+      "epoch": 0.6142857142857143,
+      "grad_norm": 0.27464860715564243,
+      "learning_rate": 6.865762593066514e-06,
+      "loss": 0.1277,
+      "step": 172
+    },
+    {
+      "epoch": 0.6178571428571429,
+      "grad_norm": 0.2714723956450917,
+      "learning_rate": 6.755890846939454e-06,
+      "loss": 0.1415,
+      "step": 173
+    },
+    {
+      "epoch": 0.6214285714285714,
+      "grad_norm": 0.28316877188473283,
+      "learning_rate": 6.646455065946386e-06,
+      "loss": 0.163,
+      "step": 174
+    },
+    {
+      "epoch": 0.625,
+      "grad_norm": 0.26976917953269747,
+      "learning_rate": 6.537469956801128e-06,
+      "loss": 0.1474,
+      "step": 175
+    },
+    {
+      "epoch": 0.6285714285714286,
+      "grad_norm": 0.20130153499017842,
+      "learning_rate": 6.428950165653204e-06,
+      "loss": 0.1058,
+      "step": 176
+    },
+    {
+      "epoch": 0.6321428571428571,
+      "grad_norm": 0.30006445926824,
+      "learning_rate": 6.320910276119576e-06,
+      "loss": 0.1557,
+      "step": 177
+    },
+    {
+      "epoch": 0.6357142857142857,
+      "grad_norm": 0.25471035440646567,
+      "learning_rate": 6.213364807324817e-06,
+      "loss": 0.1352,
+      "step": 178
+    },
+    {
+      "epoch": 0.6392857142857142,
+      "grad_norm": 0.1971014758113083,
+      "learning_rate": 6.106328211949928e-06,
+      "loss": 0.0946,
+      "step": 179
+    },
+    {
+      "epoch": 0.6428571428571429,
+      "grad_norm": 0.25877708272019345,
+      "learning_rate": 5.999814874290084e-06,
+      "loss": 0.1345,
+      "step": 180
+    },
+    {
+      "epoch": 0.6464285714285715,
+      "grad_norm": 0.2869532593978126,
+      "learning_rate": 5.893839108321584e-06,
+      "loss": 0.1485,
+      "step": 181
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 0.25488024681448673,
+      "learning_rate": 5.7884151557782305e-06,
+      "loss": 0.1411,
+      "step": 182
+    },
+    {
+      "epoch": 0.6535714285714286,
+      "grad_norm": 0.32661824559830466,
+      "learning_rate": 5.68355718423746e-06,
+      "loss": 0.1864,
+      "step": 183
+    },
+    {
+      "epoch": 0.6571428571428571,
+      "grad_norm": 0.2765974485449099,
+      "learning_rate": 5.579279285216369e-06,
+      "loss": 0.1924,
+      "step": 184
+    },
+    {
+      "epoch": 0.6607142857142857,
+      "grad_norm": 0.28641048176757467,
+      "learning_rate": 5.4755954722780236e-06,
+      "loss": 0.1401,
+      "step": 185
+    },
+    {
+      "epoch": 0.6642857142857143,
+      "grad_norm": 0.3633653450227277,
+      "learning_rate": 5.372519679148227e-06,
+      "loss": 0.2122,
+      "step": 186
+    },
+    {
+      "epoch": 0.6678571428571428,
+      "grad_norm": 0.28251549326280107,
+      "learning_rate": 5.270065757843e-06,
+      "loss": 0.153,
+      "step": 187
+    },
+    {
+      "epoch": 0.6714285714285714,
+      "grad_norm": 0.41126582781770304,
+      "learning_rate": 5.168247476807054e-06,
+      "loss": 0.1899,
+      "step": 188
+    },
+    {
+      "epoch": 0.675,
+      "grad_norm": 0.26718022299495386,
+      "learning_rate": 5.067078519063514e-06,
+      "loss": 0.1675,
+      "step": 189
+    },
+    {
+      "epoch": 0.6785714285714286,
+      "grad_norm": 0.2372115679900236,
+      "learning_rate": 4.966572480375076e-06,
+      "loss": 0.1419,
+      "step": 190
+    },
+    {
+      "epoch": 0.6821428571428572,
+      "grad_norm": 0.33710324668111635,
+      "learning_rate": 4.86674286741693e-06,
+      "loss": 0.1798,
+      "step": 191
+    },
+    {
+      "epoch": 0.6857142857142857,
+      "grad_norm": 0.3649225184724108,
+      "learning_rate": 4.767603095961652e-06,
+      "loss": 0.198,
+      "step": 192
+    },
+    {
+      "epoch": 0.6892857142857143,
+      "grad_norm": 0.312428158131683,
+      "learning_rate": 4.669166489076283e-06,
+      "loss": 0.2025,
+      "step": 193
+    },
+    {
+      "epoch": 0.6928571428571428,
+      "grad_norm": 0.2594473889879192,
+      "learning_rate": 4.571446275331903e-06,
+      "loss": 0.1474,
+      "step": 194
+    },
+    {
+      "epoch": 0.6964285714285714,
+      "grad_norm": 0.2907953351818013,
+      "learning_rate": 4.47445558702587e-06,
+      "loss": 0.1865,
+      "step": 195
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 0.279612643970489,
+      "learning_rate": 4.378207458417035e-06,
+      "loss": 0.1706,
+      "step": 196
+    },
+    {
+      "epoch": 0.7035714285714286,
+      "grad_norm": 0.25953395928309536,
+      "learning_rate": 4.282714823974088e-06,
+      "loss": 0.1473,
+      "step": 197
+    },
+    {
+      "epoch": 0.7071428571428572,
+      "grad_norm": 0.27922091609543903,
+      "learning_rate": 4.187990516637361e-06,
+      "loss": 0.1562,
+      "step": 198
+    },
+    {
+      "epoch": 0.7107142857142857,
+      "grad_norm": 0.2959893120640327,
+      "learning_rate": 4.094047266094225e-06,
+      "loss": 0.1439,
+      "step": 199
+    },
+    {
+      "epoch": 0.7142857142857143,
+      "grad_norm": 0.14884881026753682,
+      "learning_rate": 4.000897697068418e-06,
+      "loss": 0.0823,
+      "step": 200
+    },
+    {
+      "epoch": 0.7178571428571429,
+      "grad_norm": 0.3044009282702433,
+      "learning_rate": 3.908554327623425e-06,
+      "loss": 0.169,
+      "step": 201
+    },
+    {
+      "epoch": 0.7214285714285714,
+      "grad_norm": 0.3506680538543006,
+      "learning_rate": 3.817029567480228e-06,
+      "loss": 0.2161,
+      "step": 202
+    },
+    {
+      "epoch": 0.725,
+      "grad_norm": 0.3293884571998808,
+      "learning_rate": 3.7263357163496118e-06,
+      "loss": 0.1982,
+      "step": 203
+    },
+    {
+      "epoch": 0.7285714285714285,
+      "grad_norm": 0.24360834713126986,
+      "learning_rate": 3.6364849622792262e-06,
+      "loss": 0.1356,
+      "step": 204
+    },
+    {
+      "epoch": 0.7321428571428571,
+      "grad_norm": 0.33444408210986604,
+      "learning_rate": 3.5474893800157005e-06,
+      "loss": 0.1686,
+      "step": 205
+    },
+    {
+      "epoch": 0.7357142857142858,
+      "grad_norm": 0.22558400665183356,
+      "learning_rate": 3.459360929381931e-06,
+      "loss": 0.1144,
+      "step": 206
+    },
+    {
+      "epoch": 0.7392857142857143,
+      "grad_norm": 0.34424782579232677,
+      "learning_rate": 3.372111453669864e-06,
+      "loss": 0.1885,
+      "step": 207
+    },
+    {
+      "epoch": 0.7428571428571429,
+      "grad_norm": 0.2587715008045837,
+      "learning_rate": 3.2857526780488925e-06,
+      "loss": 0.1346,
+      "step": 208
+    },
+    {
+      "epoch": 0.7464285714285714,
+      "grad_norm": 0.24214032723412449,
+      "learning_rate": 3.2002962079901743e-06,
+      "loss": 0.1265,
+      "step": 209
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 0.2661742668466468,
+      "learning_rate": 3.115753527706986e-06,
+      "loss": 0.1608,
+      "step": 210
+    },
+    {
+      "epoch": 0.7535714285714286,
+      "grad_norm": 0.19602434524979756,
+      "learning_rate": 3.0321359986114096e-06,
+      "loss": 0.1186,
+      "step": 211
+    },
+    {
+      "epoch": 0.7571428571428571,
+      "grad_norm": 0.26083246797139953,
+      "learning_rate": 2.9494548577875195e-06,
+      "loss": 0.1372,
+      "step": 212
+    },
+    {
+      "epoch": 0.7607142857142857,
+      "grad_norm": 0.23778230858555124,
+      "learning_rate": 2.8677212164812464e-06,
+      "loss": 0.1423,
+      "step": 213
+    },
+    {
+      "epoch": 0.7642857142857142,
+      "grad_norm": 0.2703191684448804,
+      "learning_rate": 2.786946058607187e-06,
+      "loss": 0.155,
+      "step": 214
+    },
+    {
+      "epoch": 0.7678571428571429,
+      "grad_norm": 0.7191015403486675,
+      "learning_rate": 2.70714023927251e-06,
+      "loss": 0.2216,
+      "step": 215
+    },
+    {
+      "epoch": 0.7714285714285715,
+      "grad_norm": 0.25126851123770794,
+      "learning_rate": 2.628314483318178e-06,
+      "loss": 0.1663,
+      "step": 216
+    },
+    {
+      "epoch": 0.775,
+      "grad_norm": 0.25901386740226057,
+      "learning_rate": 2.5504793838776585e-06,
+      "loss": 0.1426,
+      "step": 217
+    },
+    {
+      "epoch": 0.7785714285714286,
+      "grad_norm": 0.4896416793964697,
+      "learning_rate": 2.473645400953366e-06,
+      "loss": 0.1819,
+      "step": 218
+    },
+    {
+      "epoch": 0.7821428571428571,
+      "grad_norm": 0.3023484206896665,
+      "learning_rate": 2.3978228600109564e-06,
+      "loss": 0.1853,
+      "step": 219
+    },
+    {
+      "epoch": 0.7857142857142857,
+      "grad_norm": 0.24572017398844886,
+      "learning_rate": 2.323021950591743e-06,
+      "loss": 0.1303,
+      "step": 220
+    },
+    {
+      "epoch": 0.7892857142857143,
+      "grad_norm": 0.33841754322388645,
+      "learning_rate": 2.249252724943336e-06,
+      "loss": 0.1467,
+      "step": 221
+    },
+    {
+      "epoch": 0.7928571428571428,
+      "grad_norm": 0.2834429062795346,
+      "learning_rate": 2.176525096668769e-06,
+      "loss": 0.1438,
+      "step": 222
+    },
+    {
+      "epoch": 0.7964285714285714,
+      "grad_norm": 0.24487797894679397,
+      "learning_rate": 2.1048488393942455e-06,
+      "loss": 0.1643,
+      "step": 223
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.22679102507137705,
+      "learning_rate": 2.0342335854556738e-06,
+      "loss": 0.1279,
+      "step": 224
+    },
+    {
+      "epoch": 0.8035714285714286,
+      "grad_norm": 0.3774325533093416,
+      "learning_rate": 1.964688824604234e-06,
+      "loss": 0.2114,
+      "step": 225
+    },
+    {
+      "epoch": 0.8071428571428572,
+      "grad_norm": 0.2091315145910457,
+      "learning_rate": 1.896223902731058e-06,
+      "loss": 0.1179,
+      "step": 226
+    },
+    {
+      "epoch": 0.8107142857142857,
+      "grad_norm": 0.22562347118461096,
+      "learning_rate": 1.8288480206112879e-06,
+      "loss": 0.1263,
+      "step": 227
+    },
+    {
+      "epoch": 0.8142857142857143,
+      "grad_norm": 0.3285000538403573,
+      "learning_rate": 1.7625702326675952e-06,
+      "loss": 0.1662,
+      "step": 228
+    },
+    {
+      "epoch": 0.8178571428571428,
+      "grad_norm": 0.2221404648237543,
+      "learning_rate": 1.6973994457534026e-06,
+      "loss": 0.1189,
+      "step": 229
+    },
+    {
+      "epoch": 0.8214285714285714,
+      "grad_norm": 0.29990632807466283,
+      "learning_rate": 1.6333444179559078e-06,
+      "loss": 0.1598,
+      "step": 230
+    },
+    {
+      "epoch": 0.825,
+      "grad_norm": 0.30132995018645475,
+      "learning_rate": 1.5704137574191202e-06,
+      "loss": 0.1747,
+      "step": 231
+    },
+    {
+      "epoch": 0.8285714285714286,
+      "grad_norm": 0.31744427301631206,
+      "learning_rate": 1.5086159211870445e-06,
+      "loss": 0.1878,
+      "step": 232
+    },
+    {
+      "epoch": 0.8321428571428572,
+      "grad_norm": 0.34224345103145587,
+      "learning_rate": 1.447959214067155e-06,
+      "loss": 0.2039,
+      "step": 233
+    },
+    {
+      "epoch": 0.8357142857142857,
+      "grad_norm": 0.3144505048078766,
+      "learning_rate": 1.3884517875143544e-06,
+      "loss": 0.1766,
+      "step": 234
+    },
+    {
+      "epoch": 0.8392857142857143,
+      "grad_norm": 0.3394594824405776,
+      "learning_rate": 1.3301016385355093e-06,
+      "loss": 0.2059,
+      "step": 235
+    },
+    {
+      "epoch": 0.8428571428571429,
+      "grad_norm": 0.20098849379680592,
+      "learning_rate": 1.2729166086147803e-06,
+      "loss": 0.114,
+      "step": 236
+    },
+    {
+      "epoch": 0.8464285714285714,
+      "grad_norm": 0.2643001377481884,
+      "learning_rate": 1.216904382659806e-06,
+      "loss": 0.1563,
+      "step": 237
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 0.34140236752735315,
+      "learning_rate": 1.1620724879689793e-06,
+      "loss": 0.1937,
+      "step": 238
+    },
+    {
+      "epoch": 0.8535714285714285,
+      "grad_norm": 0.2960179775175064,
+      "learning_rate": 1.1084282932198543e-06,
+      "loss": 0.1634,
+      "step": 239
+    },
+    {
+      "epoch": 0.8571428571428571,
+      "grad_norm": 0.28906596896070424,
+      "learning_rate": 1.0559790074789134e-06,
+      "loss": 0.1693,
+      "step": 240
+    },
+    {
+      "epoch": 0.8607142857142858,
+      "grad_norm": 0.2697212576995863,
+      "learning_rate": 1.00473167923275e-06,
+      "loss": 0.163,
+      "step": 241
+    },
+    {
+      "epoch": 0.8642857142857143,
+      "grad_norm": 0.3357321366069766,
+      "learning_rate": 9.546931954408622e-07,
+      "loss": 0.1357,
+      "step": 242
+    },
+    {
+      "epoch": 0.8678571428571429,
+      "grad_norm": 0.4160821354814617,
+      "learning_rate": 9.058702806101172e-07,
+      "loss": 0.2388,
+      "step": 243
+    },
+    {
+      "epoch": 0.8714285714285714,
+      "grad_norm": 0.2310091026767821,
+      "learning_rate": 8.582694958910809e-07,
+      "loss": 0.1495,
+      "step": 244
+    },
+    {
+      "epoch": 0.875,
+      "grad_norm": 0.5022528036104559,
+      "learning_rate": 8.118972381962853e-07,
+      "loss": 0.256,
+      "step": 245
+    },
+    {
+      "epoch": 0.8785714285714286,
+      "grad_norm": 0.29130831616869335,
+      "learning_rate": 7.667597393405602e-07,
+      "loss": 0.1776,
+      "step": 246
+    },
+    {
+      "epoch": 0.8821428571428571,
+      "grad_norm": 0.19957039218417288,
+      "learning_rate": 7.228630652035717e-07,
+      "loss": 0.0988,
+      "step": 247
+    },
+    {
+      "epoch": 0.8857142857142857,
+      "grad_norm": 0.36321388077748223,
+      "learning_rate": 6.802131149146374e-07,
+      "loss": 0.1944,
+      "step": 248
+    },
+    {
+      "epoch": 0.8892857142857142,
+      "grad_norm": 0.24595944840511677,
+      "learning_rate": 6.388156200599726e-07,
+      "loss": 0.1467,
+      "step": 249
+    },
+    {
+      "epoch": 0.8928571428571429,
+      "grad_norm": 0.21034158878551656,
+      "learning_rate": 5.986761439124289e-07,
+      "loss": 0.1126,
+      "step": 250
+    },
+    {
+      "epoch": 0.8964285714285715,
+      "grad_norm": 0.44615235095837624,
+      "learning_rate": 5.598000806838766e-07,
+      "loss": 0.2451,
+      "step": 251
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 0.21035309470955338,
+      "learning_rate": 5.221926548002876e-07,
+      "loss": 0.12,
+      "step": 252
+    },
+    {
+      "epoch": 0.9035714285714286,
+      "grad_norm": 0.2438173664233244,
+      "learning_rate": 4.858589201996433e-07,
+      "loss": 0.1458,
+      "step": 253
+    },
+    {
+      "epoch": 0.9071428571428571,
+      "grad_norm": 0.26739811417441084,
+      "learning_rate": 4.5080375965275256e-07,
+      "loss": 0.1429,
+      "step": 254
+    },
+    {
+      "epoch": 0.9107142857142857,
+      "grad_norm": 0.3093186085305965,
+      "learning_rate": 4.1703188410707087e-07,
+      "loss": 0.1756,
+      "step": 255
+    },
+    {
+      "epoch": 0.9142857142857143,
+      "grad_norm": 0.2807638356078773,
+      "learning_rate": 3.845478320536178e-07,
+      "loss": 0.1584,
+      "step": 256
+    },
+    {
+      "epoch": 0.9178571428571428,
+      "grad_norm": 0.33036425835174843,
+      "learning_rate": 3.5335596891705406e-07,
+      "loss": 0.1857,
+      "step": 257
+    },
+    {
+      "epoch": 0.9214285714285714,
+      "grad_norm": 0.22620156665010144,
+      "learning_rate": 3.2346048646903494e-07,
+      "loss": 0.1408,
+      "step": 258
+    },
+    {
+      "epoch": 0.925,
+      "grad_norm": 0.30399203832058297,
+      "learning_rate": 2.9486540226488556e-07,
+      "loss": 0.1556,
+      "step": 259
+    },
+    {
+      "epoch": 0.9285714285714286,
+      "grad_norm": 0.3113269781857376,
+      "learning_rate": 2.6757455910370486e-07,
+      "loss": 0.1921,
+      "step": 260
+    },
+    {
+      "epoch": 0.9321428571428572,
+      "grad_norm": 0.21725179306041015,
+      "learning_rate": 2.41591624511931e-07,
+      "loss": 0.1188,
+      "step": 261
+    },
+    {
+      "epoch": 0.9357142857142857,
+      "grad_norm": 0.5767021882932806,
+      "learning_rate": 2.169200902504842e-07,
+      "loss": 0.1825,
+      "step": 262
+    },
+    {
+      "epoch": 0.9392857142857143,
+      "grad_norm": 0.26446429727401966,
+      "learning_rate": 1.9356327184551716e-07,
+      "loss": 0.1438,
+      "step": 263
+    },
+    {
+      "epoch": 0.9428571428571428,
+      "grad_norm": 0.31560202017970646,
+      "learning_rate": 1.7152430814285303e-07,
+      "loss": 0.1525,
+      "step": 264
+    },
+    {
+      "epoch": 0.9464285714285714,
+      "grad_norm": 0.3529391683302704,
+      "learning_rate": 1.5080616088616884e-07,
+      "loss": 0.1869,
+      "step": 265
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 0.2725840289026022,
+      "learning_rate": 1.3141161431896809e-07,
+      "loss": 0.1797,
+      "step": 266
+    },
+    {
+      "epoch": 0.9535714285714286,
+      "grad_norm": 0.24938848004362849,
+      "learning_rate": 1.1334327481042573e-07,
+      "loss": 0.128,
+      "step": 267
+    },
+    {
+      "epoch": 0.9571428571428572,
+      "grad_norm": 0.2839629695684911,
+      "learning_rate": 9.660357050512158e-08,
+      "loss": 0.1386,
+      "step": 268
+    },
+    {
+      "epoch": 0.9607142857142857,
+      "grad_norm": 0.2881185885922197,
+      "learning_rate": 8.119475099673035e-08,
+      "loss": 0.1946,
+      "step": 269
+    },
+    {
+      "epoch": 0.9642857142857143,
+      "grad_norm": 0.23933172592888882,
+      "learning_rate": 6.711888702570556e-08,
+      "loss": 0.1289,
+      "step": 270
+    },
+    {
+      "epoch": 0.9678571428571429,
+      "grad_norm": 0.2828137522913158,
+      "learning_rate": 5.437787020100116e-08,
+      "loss": 0.1758,
+      "step": 271
+    },
+    {
+      "epoch": 0.9714285714285714,
+      "grad_norm": 0.3617417352499304,
+      "learning_rate": 4.297341274586475e-08,
+      "loss": 0.1862,
+      "step": 272
+    },
+    {
+      "epoch": 0.975,
+      "grad_norm": 0.3269777561105058,
+      "learning_rate": 3.290704726773619e-08,
+      "loss": 0.2188,
+      "step": 273
+    },
+    {
+      "epoch": 0.9785714285714285,
+      "grad_norm": 0.2943747170324779,
+      "learning_rate": 2.4180126552284523e-08,
+      "loss": 0.1864,
+      "step": 274
+    },
+    {
+      "epoch": 0.9821428571428571,
+      "grad_norm": 0.38167090302853757,
+      "learning_rate": 1.6793823381614506e-08,
+      "loss": 0.1918,
+      "step": 275
+    },
+    {
+      "epoch": 0.9857142857142858,
+      "grad_norm": 0.33959058423706057,
+      "learning_rate": 1.0749130376659366e-08,
+      "loss": 0.1504,
+      "step": 276
+    },
+    {
+      "epoch": 0.9892857142857143,
+      "grad_norm": 0.2968611418027734,
+      "learning_rate": 6.046859863781951e-09,
+      "loss": 0.186,
+      "step": 277
+    },
+    {
+      "epoch": 0.9928571428571429,
+      "grad_norm": 0.30799862242235065,
+      "learning_rate": 2.6876437656153663e-09,
+      "loss": 0.1749,
+      "step": 278
+    },
+    {
+      "epoch": 0.9964285714285714,
+      "grad_norm": 0.2512415394323519,
+      "learning_rate": 6.719335161364804e-10,
+      "loss": 0.1275,
+      "step": 279
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.3509524641302342,
+      "learning_rate": 0.0,
+      "loss": 0.1799,
+      "step": 280
+    },
+    {
+      "epoch": 1.0,
+      "step": 280,
+      "total_flos": 442411873730560.0,
+      "train_loss": 0.21834730487316847,
+      "train_runtime": 1640.8344,
+      "train_samples_per_second": 21.835,
+      "train_steps_per_second": 0.171
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 280,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 50000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 442411873730560.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ee924880feb11a92dbc5b41e61e4279f6468a4d7b2b368b687f9d456dd49129
+size 7288