Add files using upload-large-folder tool

LICENSE

@@ -0,0 +1,9 @@
+Copyright 2024 APOLLO-DEEZE-NUTZ
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+

config.json

@@ -0,0 +1,293 @@
+{
+  "architectures": [
+    "ApolloForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attn_implementation": "flash_attention_2",
+  "clip_duration": 2,
+  "drop_path_rate": 0.0,
+  "encode_batch_size": 15,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "image_aspect_ratio": "square",
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "interpolate_mode": "linear",
+  "llm_cfg": {
+    "add_cross_attention": false,
+    "architectures": [
+      "Qwen2ForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 151643,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 151645,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 3584,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 18944,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 28,
+    "min_length": 0,
+    "model_max_length": 16384,
+    "model_type": "qwen2",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 28,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 4,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 1000000.0,
+    "sep_token_id": null,
+    "sliding_window": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "tokenizer_model_max_length": 16384,
+    "tokenizer_padding_side": "right",
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 152064
+  },
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "mm_connector_cfg": {
+    "add_cross_attention": false,
+    "architectures": [
+      "Connector"
+    ],
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "ff_multi": 4,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "mm_connector",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_key_value_heads": 4,
+    "num_output_tokens": 128,
+    "num_patches": 24,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "projector_type": "mlp1x_gelu",
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "resampler_depth": 1,
+    "resampler_head_dim": 96,
+    "resampler_n_heads": 16,
+    "resampler_type": "perciver",
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "text_hidden_size": 3584,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "token_input_shape": [
+      4,
+      27,
+      27
+    ],
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "vision_hidden_size": 2560
+  },
+  "mm_connector_lr": 0.0001,
+  "mm_hidden_size": null,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "model_dtype": "torch.bfloat16",
+  "model_type": "apollo",
+  "num_attention_heads": 28,
+  "num_encode_batch": 0,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "num_video_frames": null,
+  "resume_path": "./work_dirs/final_run/apollo-Qwen2.5-7B-Instruct-internvideo2-siglip-so400m-patch14-384-freeze-perciver_128_2-newprompt-ft",
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "s2": false,
+  "s2_max_split_size": 336,
+  "s2_scales": "336,672,1008",
+  "sliding_window": null,
+  "temporal_prompt": true,
+  "timestamp_prompt": true,
+  "transformers_version": "4.44.0",
+  "tune_language_model": true,
+  "tune_mm_connector": true,
+  "tune_vision_tower": false,
+  "use_cache": true,
+  "use_mm_patch_token": false,
+  "use_mm_start_end": false,
+  "use_sliding_window": false,
+  "vision_resolution": -1,
+  "vision_tower_cfg": {
+    "add_cross_attention": false,
+    "architectures": null,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "configs": {},
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "hybrid_vision_tower",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_return_sequences": 1,
+    "num_vision_encoders": 2,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "token_output_shape": [
+      4,
+      27,
+      27
+    ],
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "vision_towers": [
+      "siglip-so400m-patch14-384",
+      "internvideo2"
+    ]
+  },
+  "vocab_size": 152064,
+  "auto_map": {
+    "AutoConfig": "configuration_apollo.ApolloConfig",
+    "AutoModelForCausalLM": "modeling_apollo.ApolloForCausalLM"
+  },
+  "model_max_length": 16384
+}

configuration_apollo.py

@@ -0,0 +1,47 @@
+#from transformers import PretrainedConfig
+from transformers import PretrainedConfig
+
+
+class ApolloConfig(PretrainedConfig):
+    model_type = "apollo"
+    def __init__(
+        self,
+        llm_cfg=None,
+        vision_tower_cfg=None,
+        mm_connector_cfg=None,
+        architectures=None,
+        resume_path=None,
+        image_aspect_ratio=None,
+        num_video_frames=None,
+        mm_vision_select_layer=None,
+        mm_vision_select_feature=None,
+        use_mm_start_end=False,
+        use_mm_patch_token=True,
+        mm_connector_lr=None,
+        vision_resolution=None,
+        interpolate_mode=None,
+        clip_duration=None,
+        vocab_size=None,
+        auto_map=None,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        
+        self.architectures = architectures
+        self.llm_cfg = llm_cfg
+        self.vision_tower_cfg = vision_tower_cfg
+        self.mm_connector_cfg = mm_connector_cfg
+        self.resume_path = resume_path
+        self.image_aspect_ratio = image_aspect_ratio
+        self.num_video_frames = num_video_frames
+        self.mm_vision_select_layer = mm_vision_select_layer
+        self.mm_vision_select_feature = mm_vision_select_feature
+        self.use_mm_start_end = use_mm_start_end
+        self.use_mm_patch_token = use_mm_patch_token
+        self.mm_connector_lr = mm_connector_lr
+        self.vision_resolution = vision_resolution
+        self.interpolate_mode = interpolate_mode
+        self.clip_duration = clip_duration
+        self.vocab_size=vocab_size
+        self.auto_map=auto_map
+

llm/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

llm/config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "./work_dirs/final_run/apollo-Qwen2.5-7B-Instruct-internvideo2-siglip-so400m-patch14-384-freeze-perciver_128_2-newprompt-ft/llm",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_max_length": 16384,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "tokenizer_model_max_length": 16384,
+  "tokenizer_padding_side": "right",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

llm/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.44.0"
+}

llm/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad28602d062e7ce6f51c4343652cef63168989c08ad1a47c11e64033c6c441ef
+size 4877660776

llm/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f392fe912c9e60fa81d1ceff9994a769f4a08f6bb63b6d92ce6ef26fbdb1704
+size 4932751008

llm/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcd8091303478c06d62188b50f1a3af122ac7bc8d2396bfda7d7a4d4d56693ec
+size 4330865200

llm/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:576065d92cfe1cfc13576cd1327672d757ef36457f4fcba9e17f0ae90a4024b7
+size 1089994880

llm/model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 15231233024
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors"
+  }
+}

llm/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

llm/tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "legacy": false,
+  "model_max_length": 16384,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

mm_connector.py

@@ -0,0 +1,306 @@
+import re, math, torch
+from collections import OrderedDict
+from typing import Optional, Tuple
+
+from torch import nn
+from torch.nn.init import trunc_normal_, normal_
+import torch.utils.checkpoint
+
+from transformers import PreTrainedModel, PretrainedConfig, AutoConfig, AutoModel
+
+
+class ClassInstantier(OrderedDict):
+    def __getitem__(self, key):
+        content = super().__getitem__(key)
+        cls, kwargs = content if isinstance(content, tuple) else (content, {})
+        return cls(**kwargs)
+
+
+ACT2CLS = {"silu": nn.SiLU}
+
+ACT2FN = ClassInstantier(ACT2CLS)
+
+
+class WeightedNorm(nn.Module):
+    def __init__(self, hidden_size):
+        """
+        WeightedNorm
+        """
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.norm = nn.LayerNorm(self.hidden_size)
+        self.wheight = nn.Parameter(torch.ones(self.hidden_size))
+        normal_(self.wheight, mean=1, std=.02)
+
+    def forward(self, x):
+        x = self.norm(x)
+        return x * self.wheight
+
+
+class PerceiverMLP(nn.Module):
+    def __init__(
+            self,
+            hidden_size: int,
+            intermediate_size: int,
+            output_size: int,
+            hidden_act: str,
+    ):
+        super().__init__()
+        self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+        self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+        self.down_proj = nn.Linear(intermediate_size, output_size, bias=False)
+        self.act_fn = ACT2FN[hidden_act]
+
+    def forward(self, x):
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+
+
+# Copied from transformers.models.llama.modeling_llama.repeat_kv
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+class PerceiverAttention(nn.Module):
+    def __init__(self, connector_config, layer_idx: Optional[int] = None) -> None:
+        """Perceiver Cross-Attention Module --> let long-form inputs be `context`, resampled embeddings be `latents`"""
+        super().__init__()
+
+        self.layer_idx = None
+        self.hidden_size = connector_config.text_hidden_size
+        self.num_heads = connector_config.resampler_n_heads
+        self.head_dim = connector_config.resampler_head_dim
+        self.num_key_value_heads = connector_config.num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+
+        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
+
+        self.is_causal = False
+
+    def forward(
+            self,
+            latents: torch.Tensor,
+            context: torch.Tensor,
+            past_key_value: Optional[Tuple[torch.Tensor]] = None,
+            output_attentions: bool = False,
+            use_cache: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        """
+        Runs Perceiver Self-Attention, with special (context, latents) appended along the `seq` dimension!
+
+        Args:
+            latents (`torch.Tensor`): Tensor of shape [bsz, n_latents, embed_dim] representing fixed length latents to compress to.
+            context (`torch.Tensor`): Tensor of shape [bsz, seq, embed_dim] representing long-form context to resample.
+            output_attentions (`bool`, *optional*, defaults to `False`): Whether to return attention weights.
+            use_cache (`bool`, *optional*, defaults to `False`): Whether to use past_key_value for caching.
+        """
+        bsz, q_len, _ = latents.size()
+        kv_seq_len = q_len + context.size()[1]
+
+        hidden_states = torch.concat([context, latents], dim=-2)
+
+        query_states = self.q_proj(latents)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, kv_seq_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, kv_seq_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        past_key_value = getattr(self, "past_key_value", past_key_value)
+
+        if past_key_value is not None:
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx)
+
+        # repeat k/v heads if n_kv_heads < n_heads
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
+
+        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.head_dim)
+
+        attn_output = self.o_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+
+PERCEIVER_ATTENTION_CLASSES = {
+    "eager": PerceiverAttention,
+}
+
+
+class PerceiverLayer(nn.Module):
+    def __init__(self, connector_config, layer_idx: int):
+        super().__init__()
+        self.hidden_size = connector_config.text_hidden_size
+        self.n_latents = connector_config.num_output_tokens
+        self.depth = connector_config.resampler_depth
+        self.ff_multi = connector_config.ff_multi
+
+        self.input_latents_norm = WeightedNorm(self.hidden_size)
+        self.input_context_norm = WeightedNorm(self.hidden_size)
+        self.self_attn = PERCEIVER_ATTENTION_CLASSES[connector_config._attn_implementation](connector_config,
+                                                                                            layer_idx=layer_idx)
+        self.post_attention_layernorm = WeightedNorm(self.hidden_size)
+        self.mlp = PerceiverMLP(
+            hidden_size=connector_config.text_hidden_size,
+            intermediate_size=connector_config.text_hidden_size * self.ff_multi,
+            output_size=connector_config.text_hidden_size,
+            hidden_act=connector_config.hidden_act,
+        )
+
+    def forward(
+            self,
+            latents: torch.Tensor,
+            context: torch.Tensor,
+            past_key_value: Optional[Tuple[torch.Tensor]] = None,
+            output_attentions: Optional[bool] = False,
+            use_cache: Optional[bool] = False,
+            **kwargs,
+    ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+        """
+        Args:
+            latents (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            context (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            use_cache (`bool`, *optional*):
+                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+                (see `past_key_values`).
+            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
+        """
+        residual = latents
+
+        latents = self.input_latents_norm(latents)
+        context = self.input_context_norm(context)
+
+        latents, self_attn_weights, present_key_value = self.self_attn(
+            latents=latents,
+            context=context,
+        )
+
+        latents = residual + latents
+        residual = latents
+
+        latents = self.post_attention_layernorm(latents)
+        latents = self.mlp(latents)
+        latents = residual + latents
+
+        outputs = (latents,)
+
+        if output_attentions:
+            outputs += (self_attn_weights,)
+
+        if use_cache:
+            outputs += (present_key_value,)
+
+        return outputs
+
+
+class PerceiverResampler(nn.Module):
+    """Perceiver Resampler that compresses input embeddings into a fixed number of latents."""
+
+    def __init__(self, connector_config) -> None:
+        super().__init__()
+        self.hidden_size = connector_config.text_hidden_size
+        self.hidden_act = connector_config.hidden_act
+        self.n_latents = connector_config.num_output_tokens
+        self.depth = connector_config.resampler_depth
+
+        # Create Latents for Perceiver
+        self.latents = nn.Parameter(torch.zeros(self.n_latents, self.hidden_size))
+
+        # Create Transformer Blocks
+        self.layers = nn.ModuleList([PerceiverLayer(connector_config, idx) for idx in range(self.depth)])
+        self.norm = WeightedNorm(self.hidden_size)
+        self._use_flash_attention_2 = connector_config._attn_implementation == "flash_attention_2"
+
+    def forward(
+            self,
+            context: torch.Tensor,
+            attention_mask: torch.Tensor = None,
+    ) -> torch.Tensor:
+        # seq embed -> bsz seq embed
+        latents = self.latents.unsqueeze(0).expand((context.shape[0], *self.latents.size()))
+
+        compressed_context = latents
+        for i, perceiver_layer in enumerate(self.layers):
+            layer_outputs = perceiver_layer(
+                compressed_context,
+                context,
+                past_key_value=None,
+                output_attentions=False,
+                use_cache=False,
+            )
+            compressed_context = layer_outputs[0]
+
+        compressed_context = self.norm(compressed_context)
+        return compressed_context
+
+
+def build_mm_projector(
+    input_dim, 
+    output_dim, 
+    projector_type, 
+    hidden_act='silu', 
+    delay_load=False, 
+    token_input_shape=0,
+    **kwargs
+    ) -> nn.Sequential:
+    
+    modules = [nn.Linear(input_dim, output_dim)]
+    mlp_gelu_match = re.match(r'.*mlp(\d+)x_gelu$', projector_type)
+    if mlp_gelu_match is not None:
+        mlp_depth = int(mlp_gelu_match.group(1))
+        for _ in range(mlp_depth - 1):
+            modules.append(nn.GELU())
+            modules.append(nn.Linear(output_dim, output_dim))
+
+    return nn.Sequential(*modules)
+
+
+class MMConnector(PreTrainedModel):
+    config_class = PretrainedConfig
+
+    def __init__(self, config: PretrainedConfig) -> None:
+        super().__init__(config)
+        self.proj = build_mm_projector(config.vision_hidden_size, config.text_hidden_size,
+                                       config.projector_type, token_input_shape=config.token_input_shape)
+        self.resampler = PerceiverResampler(config)
+
+    def forward(self, x):
+        x = self.proj(x)
+        x = self.resampler(x)
+        return x

mm_connector/config.json

@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "Connector"
+  ],
+  "attention_dropout": 0.0,
+  "ff_multi": 4,
+  "hidden_act": "silu",
+  "model_type": "mm_connector",
+  "num_key_value_heads": 4,
+  "num_output_tokens": 128,
+  "num_patches": 24,
+  "projector_type": "mlp1x_gelu",
+  "resampler_depth": 1,
+  "resampler_head_dim": 96,
+  "resampler_n_heads": 16,
+  "resampler_type": "perciver",
+  "rms_norm_eps": 1e-06,
+  "text_hidden_size": 3584,
+  "token_input_shape": [
+    4,
+    27,
+    27
+  ],
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.0",
+  "vision_hidden_size": 2560,
+  "auto_map": {
+    "AutoConfig": "configuration_connector.ConnectorConfig"
+  }
+}

mm_connector/configuration_connector.py

@@ -0,0 +1,38 @@
+import torch
+import torch.nn as nn
+from typing import Dict, List, Union
+from transformers import PreTrainedModel, PretrainedConfig, AutoConfig, AutoModel
+import torch.nn.functional as F
+import json, os
+
+
+class ConnectorConfig(PretrainedConfig):
+    model_type = "mm_connector"
+    def __init__(
+        self,
+        vision_hidden_size: List[int] = [],
+        text_hidden_size: int = 0,
+        num_patches: int = 24,
+        rms_norm_eps: float = 1e-4, 
+        token_input_shape: List[int] = [],
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.vision_hidden_size = vision_hidden_size
+        self.text_hidden_size = text_hidden_size
+        self.num_patches = num_patches
+        self.rms_norm_eps=rms_norm_eps
+        self.token_input_shape = token_input_shape
+
+    @classmethod
+    def load_config(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "ConnectorConfig":
+        cls._set_token_in_kwargs(kwargs)
+        config_dict, kwargs = cls.get_config_from_json(pretrained_model_name_or_path, **kwargs)
+        return cls.from_dict(config_dict, **kwargs)
+
+    @classmethod
+    def get_config_from_json(cls, config_file, **kwargs):
+        with open(config_file, 'r') as file:
+            config_data = json.load(file)
+        return config_data, kwargs
+

mm_connector/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2973ab0aaf61364d182eca589bdc80e28a815eb94112a83cb28d42d24da6156e
+size 355169704

modeling_apollo.py

@@ -0,0 +1,492 @@
+from typing import List, Optional, Tuple, Union
+import warnings, os, torch
+import torch.nn as nn
+
+from transformers import AutoConfig, PretrainedConfig, PreTrainedModel, AutoModelForCausalLM, AutoTokenizer
+from transformers.modeling_utils import ContextManagers, no_init_weights
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.generation.utils import GenerateOutput
+from .configuration_apollo import ApolloConfig
+
+from .vision_tower import ApolloVisionTower
+from .mm_connector import MMConnector
+
+IGNORE_INDEX = -100
+X_TOKEN_INDEX = -200
+
+
+def get_model_config(config):
+    default_keys = ["llm_cfg", "vision_tower_cfg", "mm_connector_cfg"]
+    if hasattr(config, "_name_or_path") and len(config._name_or_path) >= 2:
+        root_path = config._name_or_path
+    else:
+        root_path = config.resume_path
+
+    return_pths = []
+    for key in default_keys:
+        cfg = getattr(config, key, None)
+        if isinstance(cfg, dict):
+            try:
+                return_pths.append(os.path.join(root_path, key[:-4]))
+            except:
+                raise ValueError(f"Cannot find resume path in config for {key}!")
+        elif isinstance(cfg, PretrainedConfig):
+            return_pths.append(os.path.join(root_path, key[:-4]))
+        elif isinstance(cfg, str):
+            return_pths.append(cfg)
+
+    return_list = []
+    for pth in return_pths:
+        return_list.append(AutoConfig.from_pretrained(pth, trust_remote_code=True))
+
+    return return_list
+
+
+def build_llm_and_tokenizer(
+        llm_cfg: str,
+        config: PretrainedConfig,
+        attn_implementation=None,
+        model_max_length=None,
+        *args,
+        **kwargs,
+) -> PreTrainedModel:
+    llm_arch = getattr(llm_cfg, "architectures")[0].lower()
+    
+    llm_path = llm_cfg._name_or_path
+    llm = AutoModelForCausalLM.from_pretrained(
+        llm_path, config=llm_cfg, torch_dtype=eval(config.model_dtype), *args, **kwargs
+    )
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        llm_path,
+        model_max_length=llm_cfg.model_max_length,
+        padding_side="right",
+        use_fast=False,
+        legacy=False,
+        **kwargs
+    )
+
+    #config.hidden_size = llm.config.hidden_size
+    return llm, tokenizer
+
+
+class ApolloForCausalLM(PreTrainedModel):
+    def __init__(self, config: ApolloConfig, *args, **kwargs):
+        super().__init__(config)
+        llm_cfg, vision_tower_cfg, mm_connector_cfg = get_model_config(config)
+        model_dtype = getattr(config, "model_dtype", "torch.float16")
+        if not hasattr(config, "model_dtype"):
+            warnings.warn("model_dtype not found in config, defaulting to torch.float16.")
+            config.model_dtype = model_dtype
+        # Initialize weights and apply final processing
+
+        self.lm_head = nn.Linear(llm_cfg.hidden_size, config.vocab_size, bias=False)
+        self.vision_tower = ApolloVisionTower(config, vision_tower_cfg)
+        self.mm_connector = MMConnector.from_pretrained(mm_connector_cfg._name_or_path)
+        self.llm, self.tokenizer = build_llm_and_tokenizer(llm_cfg, config, *args, **kwargs)
+        self.post_init()
+        self.is_loaded = True
+
+    def forward(
+            self,
+            input_ids: torch.LongTensor = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            past_key_values: Optional[List[torch.FloatTensor]] = None,
+            inputs_embeds: Optional[torch.FloatTensor] = None,
+            labels: Optional[torch.LongTensor] = None,
+            use_cache: Optional[bool] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            vision_input: Optional[List[torch.FloatTensor]] = None,
+            data_types: Optional[List[str]] = None,
+            return_dict: Optional[bool] = None,
+            cache_position=None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        
+        if inputs_embeds is None:
+            (
+                input_ids,
+                position_ids,
+                attention_mask,
+                past_key_values,
+                inputs_embeds,
+                labels
+            ) = self.prepare_inputs_labels_for_multimodal(
+                input_ids,
+                position_ids,
+                attention_mask,
+                past_key_values,
+                labels,
+                vision_input,
+                data_types
+            )
+
+        return self.get_llm().forward(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            labels=labels,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+    @torch.no_grad()
+    def generate(
+            self,
+            inputs: Optional[torch.Tensor] = None,
+            vision_input: Optional[List[torch.Tensor]] = None,
+            data_types: Optional[List[str]] = None,
+            **kwargs,
+    ) -> Union[GenerateOutput, torch.LongTensor]:
+        position_ids = kwargs.pop("position_ids", None)
+        attention_mask = kwargs.pop("attention_mask", None)
+        if "inputs_embeds" in kwargs:
+            raise NotImplementedError("`inputs_embeds` is not supported")
+
+        if vision_input is not None:
+            (inputs, position_ids, attention_mask, _, inputs_embeds, _) = self.prepare_inputs_labels_for_multimodal(
+                inputs, position_ids, attention_mask, None, None, vision_input, data_types=data_types)
+        else:
+            inputs_embeds = self.embed_tokens(inputs)
+
+        return self.get_llm().generate(position_ids=position_ids, attention_mask=attention_mask,
+                                       inputs_embeds=inputs_embeds, **kwargs)
+
+    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):
+        vision_input = kwargs.pop("vision_input", None)
+        data_types = kwargs.pop("data_types", None)
+        inputs = self.get_llm().prepare_inputs_for_generation(input_ids, past_key_values=past_key_values,
+                                                              inputs_embeds=inputs_embeds, **kwargs)
+        if vision_input is not None:
+            inputs["vision_input"] = vision_input
+        if data_types is not None:
+            inputs["data_types"] = data_types
+        return inputs
+
+    @classmethod
+    def from_pretrained(
+            cls,
+            pretrained_model_name_or_path: Optional[Union[str, os.PathLike]],
+            *model_args,
+            config: Optional[Union[PretrainedConfig, str, os.PathLike]] = None,
+            cache_dir: Optional[Union[str, os.PathLike]] = None,
+            ignore_mismatched_sizes: bool = False,
+            force_download: bool = False,
+            local_files_only: bool = False,
+            token: Optional[Union[str, bool]] = None,
+            revision: str = "main",
+            use_safetensors: bool = None,
+            **kwargs,
+    ):
+
+        return cls.load_pretrained(
+            pretrained_model_name_or_path,
+            *model_args,
+            config=config,
+            cache_dir=cache_dir,
+            ignore_mismatched_sizes=ignore_mismatched_sizes,
+            force_download=force_download,
+            local_files_only=local_files_only,
+            token=token,
+            revision=revision,
+            use_safetensors=use_safetensors,
+            **kwargs,
+        )
+
+    def get_llm(self):
+        return self.llm
+
+    def get_vision_tower(self):
+        return self.vision_tower
+
+    def get_mm_connector(self):
+        return self.mm_connector
+
+    @classmethod
+    def load_pretrained(cls, model_path_or_config, *args, **kwargs):
+        kwargs.pop("config", None)
+        
+        if isinstance(model_path_or_config, str):
+            config = AutoConfig.from_pretrained(model_path_or_config, trust_remote_code=True, **kwargs)
+        elif isinstance(model_path_or_config, ApolloConfig):
+            config = model_path_or_config
+        else:
+            raise NotImplementedError(f"wrong type, {type(model_path_or_config)} \
+                                      {isinstance(model_path_or_config, ApolloConfig)}")
+
+        model_dtype = getattr(config, "model_dtype", "torch.float16")
+        if not hasattr(config, "model_dtype"):
+            warnings.warn("model_dtype not found in config, defaulting to torch.float16.")
+            config.model_dtype = model_dtype
+
+        with ContextManagers([no_init_weights(_enable=True), ]):
+            vlm = cls(config, *args, **kwargs)
+
+        if hasattr(vlm, "llm") and hasattr(vlm, "vision_tower") and hasattr(vlm, "mm_connector"):
+            if vlm.is_loaded:
+                return vlm
+            else:
+                print('loading model failed!')
+        else:
+            print('loading model failed!')
+
+    def _encode_mm(self, x):
+        x = self.get_vision_tower()(x)
+        x = self.mm_connector(x)
+        return x
+
+    def encode_mm_minibatch(self, x):
+        split_sizes = [x_s[0].shape[0] for x_s in x]
+        x = [torch.split(torch.cat([x_s[i] for x_s in x], dim=0), self.config.encode_batch_size) for i in
+             range(self.get_vision_tower().num_vision_encoders)]
+        swapped_x = []
+        for i in range(len(x[0])):
+            swapped_x.append([x_s[i] for x_s in x])
+
+        features = []
+        for xx in swapped_x:
+            xx = self._encode_mm(xx)
+            features.append(xx)
+        x = torch.cat(features, dim=0)
+        x = torch.split(x, split_sizes, dim=0)
+        return [xx.contiguous().view(-1, xx.shape[2]) for xx in x]
+
+    def prepare_inputs_labels_for_multimodal(
+            self, input_ids, position_ids, attention_mask, past_key_values, labels, vision_input, data_types
+    ):
+        vision_tower = self.get_vision_tower()
+        if vision_tower is None or vision_input is None or input_ids.shape[1] == 1:
+            if (
+                    past_key_values is not None
+                    and vision_tower is not None
+                    and vision_input is not None
+                    and input_ids.shape[1] == 1
+            ):
+                target_shape = past_key_values[-1][-1].shape[-2] + 1
+                attention_mask = torch.cat(
+                    (
+                        attention_mask,
+                        torch.ones(
+                            (
+                                attention_mask.shape[0],
+                                target_shape - attention_mask.shape[1],
+                            ),
+                            dtype=attention_mask.dtype,
+                            device=attention_mask.device,
+                        ),
+                    ),
+                    dim=1,
+                )
+                position_ids = torch.sum(attention_mask, dim=1).unsqueeze(-1) - 1
+            return (
+                input_ids,
+                position_ids,
+                attention_mask,
+                past_key_values,
+                None,
+                labels,
+            )
+
+        '''
+            vision_input is a list of tuples, and data_type is a list of strings:
+            data_type = ['image', 'video', 'video'..., 'text']
+            (for one video and two image encoders)
+            vision_input = 
+            [
+                [image(1, T, C, H, W), image(1, T, C, H, W), image(1, T, C, H, W)],
+                [video(Nc1, C, T, H, W), video(Nc1, T, C, H, W), video(Nc1, T, C, H, W)],
+                [video(Nc2, C, T, H, W), video(Nc2, T, C, H, W), video(Nc2, T, C, H, W)],
+            ]
+            -> video encoders typlically expect (C,T,H,W), images expect (C,H,W).
+        '''
+        # ====================================================================================================
+        merged_mm_features = self.encode_mm_minibatch(vision_input)
+
+        if not getattr(self.config, "tune_language_model", True) and getattr(self.config, "use_mm_start_end", False):
+            raise NotImplementedError
+        # ====================================================================================================
+        # Let's just add dummy tensors if they do not exist,
+        # it is a headache to deal with None all the time.
+        # But it is not ideal, and if you have a better idea,
+        # please open an issue / submit a PR, thanks.
+        _labels = labels
+        _position_ids = position_ids
+        _attention_mask = attention_mask
+        if attention_mask is None:
+            attention_mask = torch.ones_like(input_ids, dtype=torch.bool)
+        else:
+            attention_mask = attention_mask.bool()
+        if position_ids is None:
+            position_ids = torch.arange(0, input_ids.shape[1], dtype=torch.long, device=input_ids.device)
+        if labels is None:
+            labels = torch.full_like(input_ids, IGNORE_INDEX)
+
+        # remove the padding using attention_mask
+        input_ids_copy = input_ids.clone()
+        # kentang-mit@: Otherwise tokenizer out of bounds. Embeddings of image tokens will not be used.
+        input_ids_copy[input_ids_copy == X_TOKEN_INDEX] = 0
+        input_embeds = self.get_llm().model.embed_tokens(input_ids_copy)
+
+        input_ids = [
+            cur_input_ids[cur_attention_mask] for cur_input_ids, cur_attention_mask in zip(input_ids, attention_mask)
+        ]
+        input_embeds_1 = [
+            cur_input_embeds[cur_attention_mask]
+            for cur_input_embeds, cur_attention_mask in zip(input_embeds, attention_mask)
+        ]
+        labels = [cur_labels[cur_attention_mask] for cur_labels, cur_attention_mask in zip(labels, attention_mask)]
+        # input_ids, new_input_embeds = self.inputs_merger(input_ids, input_embeds_1, merged_mm_features)
+        new_labels = []
+        new_input_embeds = []
+        # print("BEFORE BATCH LOOP:", len(input_ids), input_ids[0].shape, input_ids[0].device, [(x == X_TOKEN_INDEX).sum() for x in input_ids])
+        # kentang-mit@: If some part of the model is executed in the loop, the the loop length needs to be a constant.
+        for batch_idx, (cur_labels, cur_input_ids, mm_features) in enumerate(
+                zip(labels, input_ids, merged_mm_features)):
+            cur_input_ids = input_ids[batch_idx]
+            num_mm = (cur_input_ids == X_TOKEN_INDEX).sum()
+            if num_mm == 0:
+                cur_input_embeds_1 = input_embeds_1[batch_idx]
+                cur_input_embeds = torch.cat([cur_input_embeds_1, mm_features[0:0]], dim=0)
+                new_input_embeds.append(cur_input_embeds)
+                new_labels.append(cur_labels)
+                # kenang-mit@: we do not have placeholdr image for text-only data now.
+                continue
+
+            if mm_features.shape[0] != num_mm:
+                print(data_types[batch_idx])
+                assert num_mm == len(
+                    mm_features), f'Error in {data_types[batch_idx]}{num_mm}=/={len(mm_features)} not the same number of vision tokens in and vision embeddings!'
+
+            cur_input_embeds = input_embeds_1[batch_idx]
+            image_token_indices = (
+                    [-1] + torch.where(cur_input_ids == X_TOKEN_INDEX)[0].tolist() + [cur_input_ids.shape[0]]
+            )
+            cur_input_ids_noim = []
+            cur_labels = labels[batch_idx]
+            cur_labels_noim = []
+            cur_input_embeds_no_im = []
+            for i in range(len(image_token_indices) - 1):
+                cur_input_ids_noim.append(cur_input_ids[image_token_indices[i] + 1: image_token_indices[i + 1]])
+                cur_labels_noim.append(cur_labels[image_token_indices[i] + 1: image_token_indices[i + 1]])
+                cur_input_embeds_no_im.append(cur_input_embeds[image_token_indices[i] + 1: image_token_indices[i + 1]])
+
+            cur_new_input_embeds = []
+            cur_new_labels = []
+            for i in range(num_mm + 1):
+                cur_new_input_embeds.append(cur_input_embeds_no_im[i])
+                # print("cur_new_input_embeds1", cur_new_input_embeds.shape[-1])
+                cur_new_labels.append(cur_labels_noim[i])
+                if i < num_mm:
+                    cur_image_features = mm_features[i:i + 1]
+                    cur_new_input_embeds.append(cur_image_features)
+                    # print("cur_new_input_embeds2", cur_new_input_embeds.shape[-1])
+                    cur_new_labels.append(
+                        torch.full(
+                            (cur_image_features.shape[0],),
+                            IGNORE_INDEX,
+                            device=cur_labels.device,
+                            dtype=cur_labels.dtype,
+                        )
+                    )
+
+            cur_new_input_embeds = torch.cat(cur_new_input_embeds)
+            cur_new_labels = torch.cat(cur_new_labels)
+
+            new_input_embeds.append(cur_new_input_embeds)
+            new_labels.append(cur_new_labels)
+
+        # Truncate sequences to max length as image embeddings can make the sequence longer
+        tokenizer_model_max_length = getattr(self.get_llm().config, "tokenizer_model_max_length", None)
+        if tokenizer_model_max_length is not None:
+            if any(len(x) > tokenizer_model_max_length for x in new_input_embeds):
+                priny("Inputs truncated!")
+            new_input_embeds = [x[:tokenizer_model_max_length] for x in new_input_embeds]
+            new_labels = [x[:tokenizer_model_max_length] for x in new_labels]
+        # Combine them
+        max_len = max(x.shape[0] for x in new_input_embeds)
+        batch_size = len(new_input_embeds)
+
+        new_input_embeds_padded = []
+        new_labels_padded = torch.full(
+            (batch_size, max_len),
+            IGNORE_INDEX,
+            dtype=new_labels[0].dtype,
+            device=new_labels[0].device,
+        )
+        attention_mask = torch.zeros(
+            (batch_size, max_len),
+            dtype=attention_mask.dtype,
+            device=attention_mask.device,
+        )
+        position_ids = torch.zeros((batch_size, max_len), dtype=position_ids.dtype, device=position_ids.device)
+        for i, (cur_new_embed, cur_new_labels) in enumerate(zip(new_input_embeds, new_labels)):
+            cur_len = cur_new_embed.shape[0]
+            if getattr(self.get_llm().config, "tokenizer_padding_side", "right") == "left":
+                new_input_embeds_padded.append(
+                    torch.cat(
+                        (
+                            torch.zeros(
+                                (max_len - cur_len, cur_new_embed.shape[1]),
+                                dtype=cur_new_embed.dtype,
+                                device=cur_new_embed.device,
+                            ),
+                            cur_new_embed,
+                        ),
+                        dim=0,
+                    )
+                )
+                if cur_len > 0:
+                    new_labels_padded[i, -cur_len:] = cur_new_labels
+                    attention_mask[i, -cur_len:] = True
+                    position_ids[i, -cur_len:] = torch.arange(
+                        0, cur_len, dtype=position_ids.dtype, device=position_ids.device
+                    )
+            else:
+                new_input_embeds_padded.append(
+                    torch.cat(
+                        (
+                            cur_new_embed,
+                            torch.zeros(
+                                (max_len - cur_len, cur_new_embed.shape[1]),
+                                dtype=cur_new_embed.dtype,
+                                device=cur_new_embed.device,
+                            ),
+                        ),
+                        dim=0,
+                    )
+                )
+                if cur_len > 0:
+                    new_labels_padded[i, :cur_len] = cur_new_labels
+                    attention_mask[i, :cur_len] = True
+                    position_ids[i, :cur_len] = torch.arange(
+                        0, cur_len, dtype=position_ids.dtype, device=position_ids.device
+                    )
+
+        new_input_embeds = torch.stack(new_input_embeds_padded, dim=0)
+
+        if _labels is None:
+            new_labels = None
+        else:
+            new_labels = new_labels_padded
+
+        if _attention_mask is None:
+            attention_mask = None
+        else:
+            attention_mask = attention_mask.to(dtype=_attention_mask.dtype)
+
+        if _position_ids is None:
+            position_ids = None
+
+        return (
+            None,
+            position_ids,
+            attention_mask,
+            past_key_values,
+            new_input_embeds,
+            new_labels,
+        )

vision_tower.py

@@ -0,0 +1,556 @@
+import torch, os, PIL, numbers
+from PIL import Image
+import cv2
+
+from transformers.modeling_utils import PreTrainedModel
+from transformers.models.siglip.modeling_siglip import SiglipVisionModel
+from transformers import AutoConfig, AutoModel, SiglipImageProcessor, SiglipVisionConfig, PretrainedConfig
+from typing import Union
+import torch.nn.functional as F
+import numpy as np
+
+
+def crop_clip(clip, min_h, min_w, h, w):
+    if isinstance(clip[0], np.ndarray):
+        cropped = [img[min_h:min_h + h, min_w:min_w + w, :] for img in clip]
+
+    elif isinstance(clip[0], PIL.Image.Image):
+        cropped = [
+            img.crop((min_w, min_h, min_w + w, min_h + h)) for img in clip
+        ]
+    else:
+        raise TypeError('Expected numpy.ndarray or PIL.Image' +
+                        'but got list of {0}'.format(type(clip[0])))
+    return cropped
+
+
+class Normalize(object):
+    """Normalize a clip with mean and standard deviation.
+    Given mean: ``(M1,...,Mn)`` and std: ``(S1,..,Sn)`` for ``n`` channels, this transform
+    will normalize each channel of the input ``torch.*Tensor`` i.e.
+    ``input[channel] = (input[channel] - mean[channel]) / std[channel]``
+    .. note::
+        This transform acts out of place, i.e., it does not mutates the input tensor.
+    Args:
+        mean (sequence): Sequence of means for each channel.
+        std (sequence): Sequence of standard deviations for each channel.
+    """
+
+    def __init__(self, mean, std):
+        self.mean = mean
+        self.std = std
+
+    def __call__(self, clip):
+        """
+        Args:
+            clip (Tensor): Tensor clip of size (T, C, H, W) to be normalized.
+        Returns:
+            Tensor: Normalized Tensor clip.
+        """
+        return normalize(clip, self.mean, self.std)
+
+    def __repr__(self):
+        return self.__class__.__name__ + '(mean={0}, std={1})'.format(self.mean, self.std)
+
+
+class CenterCrop(object):
+    """Extract center crop at the same location for a list of images
+    Args:
+    size (sequence or int): Desired output size for the
+    crop in format (h, w)
+    """
+
+    def __init__(self, size):
+        if isinstance(size, numbers.Number):
+            size = (size, size)
+
+        self.size = size
+
+    def __call__(self, clip):
+        """
+        Args:
+        img (PIL.Image or numpy.ndarray): List of images to be cropped
+        in format (h, w, c) in numpy.ndarray
+        Returns:
+        PIL.Image or numpy.ndarray: Cropped list of images
+        """
+        h, w = self.size
+        if isinstance(clip[0], np.ndarray):
+            im_h, im_w, im_c = clip[0].shape
+        elif isinstance(clip[0], PIL.Image.Image):
+            im_w, im_h = clip[0].size
+        else:
+            raise TypeError('Expected numpy.ndarray or PIL.Image' +
+                            'but got list of {0}'.format(type(clip[0])))
+        if w > im_w or h > im_h:
+            error_msg = (
+                'Initial image size should be larger then '
+                'cropped size but got cropped sizes : ({w}, {h}) while '
+                'initial image is ({im_w}, {im_h})'.format(
+                    im_w=im_w, im_h=im_h, w=w, h=h))
+            raise ValueError(error_msg)
+
+        x1 = int(round((im_w - w) / 2.))
+        y1 = int(round((im_h - h) / 2.))
+        cropped = crop_clip(clip, y1, x1, h, w)
+
+        return cropped
+
+
+def resize_clip(clip, size, interpolation='bilinear'):
+    if isinstance(clip[0], np.ndarray):
+        if isinstance(size, numbers.Number):
+            im_h, im_w, im_c = clip[0].shape
+            # Min spatial dim already matches minimal size
+            if (im_w <= im_h and im_w == size) or (im_h <= im_w
+                                                   and im_h == size):
+                return clip
+            new_h, new_w = get_resize_sizes(im_h, im_w, size)
+            size = (new_w, new_h)
+        else:
+            size = size[0], size[1]
+        if interpolation == 'bilinear':
+            np_inter = cv2.INTER_LINEAR
+        else:
+            np_inter = cv2.INTER_NEAREST
+        scaled = [
+            cv2.resize(img, size, interpolation=np_inter) for img in clip
+        ]
+    elif isinstance(clip[0], PIL.Image.Image):
+        if isinstance(size, numbers.Number):
+            im_w, im_h = clip[0].size
+            # Min spatial dim already matches minimal size
+            if (im_w <= im_h and im_w == size) or (im_h <= im_w
+                                                   and im_h == size):
+                return clip
+            new_h, new_w = get_resize_sizes(im_h, im_w, size)
+            size = (new_w, new_h)
+        else:
+            size = size[1], size[0]
+        if interpolation == 'bilinear':
+            pil_inter = PIL.Image.BILINEAR
+        else:
+            pil_inter = PIL.Image.NEAREST
+        scaled = [img.resize(size, pil_inter) for img in clip]
+    else:
+        raise TypeError('Expected numpy.ndarray or PIL.Image' +
+                        'but got list of {0}'.format(type(clip[0])))
+    return scaled
+
+
+def _is_tensor_clip(clip):
+    return torch.is_tensor(clip) and clip.ndimension() == 4
+
+
+def get_resize_sizes(im_h, im_w, size):
+    if im_w < im_h:
+        ow = size
+        oh = int(size * im_h / im_w)
+    else:
+        oh = size
+        ow = int(size * im_w / im_h)
+    return oh, ow
+
+
+def normalize(clip, mean, std, inplace=False):
+    if not _is_tensor_clip(clip):
+        raise TypeError('tensor is not a torch clip.')
+
+    if not inplace:
+        clip = clip.clone()
+
+    dtype = clip.dtype
+    mean = torch.as_tensor(mean, dtype=dtype, device=clip.device)
+    std = torch.as_tensor(std, dtype=dtype, device=clip.device)
+    clip.sub_(mean[:, None, None, None]).div_(std[:, None, None, None])
+
+    return clip
+
+
+class Resize(object):
+    """Resizes a list of (H x W x C) numpy.ndarray to the final size
+    The larger the original image is, the more times it takes to
+    interpolate
+    Args:
+    interpolation (str): Can be one of 'nearest', 'bilinear'
+    defaults to nearest
+    size (tuple): (widht, height)
+    """
+
+    def __init__(self, size, interpolation='nearest'):
+        self.size = size
+        self.interpolation = interpolation
+
+    def __call__(self, clip):
+        resized = resize_clip(
+            clip, self.size, interpolation=self.interpolation)
+        return resized
+
+
+class Compose(object):
+    """Composes several transforms
+    Args:
+    transforms (list of ``Transform`` objects): list of transforms
+    to compose
+    """
+
+    def __init__(self, transforms):
+        self.transforms = transforms
+
+    def __call__(self, clip):
+        for t in self.transforms:
+            clip = t(clip)
+        return clip
+
+
+def convert_img(img):
+    """Converts (H, W, C) numpy.ndarray to (C, W, H) format"""
+    if len(img.shape) == 3:
+        img = img.transpose(2, 0, 1)
+    if len(img.shape) == 2:
+        img = np.expand_dims(img, 0)
+    return img
+
+
+class ClipToTensor(object):
+    """Convert a list of m (H x W x C) numpy.ndarrays in the range [0, 255]
+    to a torch.FloatTensor of shape (C x m x H x W) in the range [0, 1.0]
+    """
+
+    def __init__(self, channel_nb=3, div_255=True, numpy=False):
+        self.channel_nb = channel_nb
+        self.div_255 = div_255
+        self.numpy = numpy
+
+    def __call__(self, clip):
+        """
+        Args: clip (list of numpy.ndarray): clip (list of images)
+        to be converted to tensor.
+        """
+        # Retrieve shape
+        if isinstance(clip[0], np.ndarray):
+            h, w, ch = clip[0].shape
+            assert ch == self.channel_nb, "Got {0} instead of 3 channels".format(ch)
+        elif isinstance(clip[0], Image.Image):
+            w, h = clip[0].size
+        else:
+            raise TypeError(
+                "Expected numpy.ndarray or PIL.Image\
+            but got list of {0}".format(
+                    type(clip[0])
+                )
+            )
+
+        np_clip = np.zeros([self.channel_nb, len(clip), int(h), int(w)])
+
+        # Convert
+        for img_idx, img in enumerate(clip):
+            if isinstance(img, np.ndarray):
+                pass
+            elif isinstance(img, Image.Image):
+                img = np.array(img, copy=False)
+            else:
+                raise TypeError(
+                    "Expected numpy.ndarray or PIL.Image\
+                but got list of {0}".format(
+                        type(clip[0])
+                    )
+                )
+            img = convert_img(img)
+            np_clip[:, img_idx, :, :] = img
+        if self.numpy:
+            if self.div_255:
+                np_clip = np_clip / 255.0
+            return np_clip
+
+        else:
+            tensor_clip = torch.from_numpy(np_clip)
+
+            if not isinstance(tensor_clip, torch.FloatTensor):
+                tensor_clip = tensor_clip.float()
+            if self.div_255:
+                tensor_clip = torch.div(tensor_clip, 255)
+            return tensor_clip
+
+
+class VisionTowerConfig(PretrainedConfig):
+    model_type = "vision_tower"
+
+    def __init__(self, vision_tower_name: str = None, **kwargs):
+        super().__init__()
+        self.vision_tower_name = vision_tower_name
+
+
+class ProcessorWrapper:
+    def __init__(self, transform=None, processor=None, height=378, width=378, frames_per_clip=1,
+                 image_mean=[0.48145466, 0.4578275, 0.40821073]):
+        assert transform is not None or processor is not None, "ERROR: you did not define both `transform` and `processor`! You must define either transform or processor"
+        assert transform is None or processor is None, "ERROR: you did defined both `transform` and `processor`! You must define only one of: transform or processor"
+        self._size = {
+            "height": height,
+            "width": width,
+            "frames_per_clip": frames_per_clip
+        }
+        self._transforms = transform
+        self._processor = processor
+        self.image_mean = image_mean
+
+    @property
+    def size(self):
+        return self._size
+
+    def preprocess(self, image, return_tensors='pt'):
+        # Ensure image is a PIL Image
+        output = {}
+        if self._transforms is not None:
+            output['pixel_values'] = [self._transforms(image)]
+
+        else:
+            output = self._processor(image, return_tensors='pt')
+        return output
+
+    def save_pretrained(self, save_path):
+        if self._transforms is not None:
+            transform_dict = transform_to_dict(self._transforms)
+            transform_dict["image_processor_type"] = "transforms"
+            with open(os.path.join(save_path, 'preprocessor_config.json'), 'w') as f:
+                json.dump(transform_dict, f, indent=4)
+        else:
+            self._processor.save_pretrained(save_path)
+        return
+
+
+class VisionTower(PreTrainedModel):
+    config_class = VisionTowerConfig
+
+    def __init__(self, model_name_or_path: str, config: PretrainedConfig, vision_config: VisionTowerConfig = None):
+        super().__init__(vision_config)
+        self.vision_tower_name = model_name_or_path
+        self.vision_config = vision_config
+        self.select_layer = getattr(config, "mm_vision_select_layer", -2)
+        self.select_feature = getattr(config, "mm_vision_select_feature", "patch")
+        self.encode_batch_size = getattr(config, "encode_batch_size", 0) // 2
+        self.num_encode_batch = getattr(config, "num_encode_batch", 0) // 2
+        self.temporal_tubelet_size = getattr(vision_config, "tubelet_size", 1)
+
+    def feature_select(self, image_features):
+        if self.select_layer is not None:
+            image_features = image_features.hidden_states[self.select_layer]
+            
+        if self.select_feature == "patch":
+            image_features = image_features[:, 1:]
+        elif self.select_feature == "cls_patch":
+            image_features = image_features
+        else:
+            raise ValueError(f"Unexpected select feature: {self.select_feature}")
+            
+        return image_features
+
+    def vision_tower_forward(self, image):
+        image_feature = self.vision_tower(image, output_hidden_states=True)
+        return image_feature
+    
+    def _forward(self, images, out_T=1):
+        if type(images) is list:
+            image_features = []
+            for image in images:
+                image_feature = self.vision_tower_forward(image.to(device=self.device, dtype=self.dtype).unsqueeze(0))
+                image_feature = self.feature_select(image_feature).to(image.dtype)
+                image_feature = image_features.reshape(image_feature.shape[0], self.W, self.H, self.D)
+                image_features.append(image_feature)
+        else:
+            original_shape = images.shape
+            if len(original_shape) == 5 and self.T == 1:
+                # downsample temporally if needed, and reshape from (B, T, C, W, H) to (B*T, C, W, H).
+                images = images[:, ::original_shape[1] // out_T, ...]
+                original_shape = images.shape
+                images = images.view(-1, *original_shape[2:])
+
+            image_features = self.vision_tower_forward(images.to(device=self.device, dtype=self.dtype))
+            image_features = self.feature_select(image_features).to(images.dtype)
+            # Reshape back to (B, T, ...) if necessary
+            if len(original_shape) == 5 and self.T == 1:
+                # Assuming the feature dimension does not change, adapt the following line if it does
+                new_shape = list(image_features.shape[:-2]) + [self.W, self.H, self.hidden_size]
+                image_features = image_features.reshape(new_shape)
+                feature_size = image_features.shape[1:]
+                image_features = image_features.view(original_shape[0], original_shape[1], *feature_size)
+                
+            else:
+                image_features = image_features.reshape(image_features.shape[0], self.T, self.W, self.H, self.hidden_size)
+                
+        return image_features
+    
+    def forward(self, images):
+        return self._forward(images)
+
+    @property
+    def dummy_feature(self):
+        return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)
+
+    @property
+    def dtype(self):
+        return self.vision_tower.dtype
+
+    @property
+    def device(self):
+        return self.vision_tower.device
+
+    @property
+    def num_patches(self):
+        return (self.config.image_size // self.config.patch_size) ** 2
+
+
+class InternVideoTower(VisionTower):
+    def __init__(self, model_name_or_path: str, config: PretrainedConfig, vision_config: PretrainedConfig = None):
+        if vision_config is None:
+            vision_config = AutoConfig.from_pretrained(model_name_or_path, trust_remote_code=True)
+
+        super().__init__(model_name_or_path, config, vision_config)
+        self.vision_config = vision_config
+        normalize = ((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
+
+        print('loading: ', model_name_or_path)
+        model = AutoModel.from_pretrained(model_name_or_path, trust_remote_code=True)
+        self.vision_tower = model.to(dtype=eval(config.model_dtype))
+
+        transform = Compose([
+            Resize(self.vision_config.img_size, interpolation='bilinear'),
+            CenterCrop(size=(self.vision_config.img_size, self.vision_config.img_size)),
+            ClipToTensor(),
+            Normalize(mean=normalize[0], std=normalize[1])
+        ])
+
+        self.vision_processor = ProcessorWrapper(transform=transform,
+                                                 height=self.vision_config.img_size,
+                                                 width=self.vision_config.img_size,
+                                                 frames_per_clip=self.vision_config.num_frames,
+                                                 image_mean=normalize[0])
+
+        self.W = self.H = vision_config.img_size // vision_config.patch_size
+        self.T = self.vision_config.num_frames // self.vision_config.tubelet_size
+        self.num_frames = self.vision_config.num_frames
+        self.hidden_size = vision_config.d_model
+        self.vision_select_layer=self.select_layer
+        self.select_layer=None
+
+    def vision_tower_forward(self, video):
+        if video.shape[-3] < self.num_frames:
+            video = video.repeat_interleave(self.num_frames, dim=-3)
+        elif video.shape[-3] > self.num_frames:
+            video = video[:, :, ::video.shape[-3] // self.num_frames, ...]
+
+        video_feature = self.vision_tower(video.to(device=self.device, dtype=self.dtype),
+                                          x_vis_return_idx=self.vision_select_layer, x_vis_only=True)
+        
+        return video_feature
+
+    @property
+    def device(self):
+        return self.vision_tower.pos_embed.device
+
+
+class SiglipVisionTower(VisionTower):
+    def __init__(self, model_name_or_path: str, config: PretrainedConfig, vision_config: PretrainedConfig = None):
+        if vision_config is None:
+            vision_config = SiglipVisionConfig.from_pretrained(model_name_or_path)
+
+        super().__init__(model_name_or_path, config, vision_config)
+        self.vision_config = vision_config
+        self.vision_tower_name = model_name_or_path
+        self.vision_processor = SiglipImageProcessor.from_pretrained(self.vision_tower_name)
+
+        print('loading: ', model_name_or_path)
+        self.vision_tower = SiglipVisionModel.from_pretrained(self.vision_tower_name)
+
+        self.hidden_size = self.vision_config.hidden_size
+        self.W = self.H = self.vision_config.image_size // self.vision_config.patch_size
+        self.T = 1
+        self.select_feature = "cls_patch"
+
+
+class ApolloVisionTower(PreTrainedModel):
+    def __init__(self, config, vision_tower_cfg):
+        super(ApolloVisionTower, self).__init__(config, vision_tower_cfg)
+        self.model_name_or_path = vision_tower_cfg._name_or_path
+        self.vision_towers = vision_tower_cfg.vision_towers
+        self._config = vision_tower_cfg
+
+        for vision_tower_name in self.vision_towers:
+            if 'internvideo' in vision_tower_name.lower():
+                vision_tower = InternVideoTower(os.path.join(vision_tower_cfg._name_or_path, vision_tower_name), config)
+            elif 'siglip' in vision_tower_name.lower():
+                vision_tower = SiglipVisionTower(os.path.join(vision_tower_cfg._name_or_path, vision_tower_name),
+                                                 config)
+
+            setattr(self, vision_tower_name, vision_tower)
+
+        self.vision_processor = [getattr(self, vt).vision_processor for vt in self.vision_towers]
+        self.num_vision_encoders = len(self.vision_towers)
+        self.W = self.H = max([getattr(self, vt).W for vt in self.vision_towers])
+        self.T = max([getattr(self, vt).T for vt in self.vision_towers])
+        self.max_tubelet_size = max(
+            [getattr(getattr(self, vt).vision_config, 'tubelet_size', 1) for vt in self.vision_towers])
+        
+        self._hidden_size = sum([getattr(self, vt).hidden_size for vt in self.vision_towers])
+        self.token_output_shape = (self.T, self.W, self.H)
+        self.config.num_vision_encoders = self.num_vision_encoders
+        self.config.vision_towers = self.vision_towers
+        self.config.token_output_shape = self.token_output_shape
+
+    def forward(self, x):
+        output_features = []
+        for x_s, vision_tower_name in zip(x, self.vision_towers):
+            vision_tower = getattr(self, vision_tower_name)
+            features = vision_tower._forward(x_s, out_T=self.T)
+
+            if len(features.shape) != len(self.token_output_shape) + 2:
+                features = features.unsqueeze(1)
+
+            if features.shape[-len(self.token_output_shape) - 1:-1] != self.token_output_shape:
+                features = features.permute(0, 4, 1, 2, 3).contiguous()  # shape [B, D, T, W, H]
+                features = F.interpolate(features.to(torch.float32), size=self.token_output_shape, mode='trilinear',
+                                         align_corners=False).to(features.dtype)
+                features = features.permute(0, 2, 3, 4, 1).contiguous()
+
+            output_features.append(features)
+
+        output_features = torch.cat(output_features, dim=-1)
+        output_features = torch.flatten(output_features, start_dim=1, end_dim=-2)
+        return output_features
+
+    def save_pretrained(
+            self,
+            save_directory: Union[str, os.PathLike],
+            state_dict=None,
+            **kwargs,
+    ):
+        if state_dict is None:
+            state_dict = self.state_dict()
+
+        for vision_tower_name in self.vision_towers:
+            vision_tower = getattr(self, vision_tower_name)
+            vision_tower_state_dict = OrderedDict(
+                {k.split(f"vision_tower.{vision_tower_name}.vision_tower.")[-1]: v for k, v in state_dict.items() if
+                 vision_tower_name in k}
+            )
+            vision_tower.vision_tower.save_pretrained(os.path.join(save_directory, vision_tower_name),
+                                                      state_dict=vision_tower_state_dict, **kwargs)
+            vision_tower.vision_processor.save_pretrained(os.path.join(save_directory, vision_tower_name))
+
+        config = self.config
+        config.configs = {}
+        config.save_pretrained(save_directory)
+
+    @property
+    def patch_size(self):
+        return self._patch_size
+
+    @property
+    def image_size(self):
+        return self._image_size
+
+    @property
+    def hidden_size(self):
+        return self._hidden_size
+

vision_tower/config.json

@@ -0,0 +1,18 @@
+{
+  "configs": {},
+  "model_type": "hybrid_vision_tower",
+  "num_vision_encoders": 2,
+  "token_output_shape": [
+    4,
+    27,
+    27
+  ],
+  "transformers_version": "4.44.0",
+  "vision_towers": [
+    "siglip-so400m-patch14-384",
+    "internvideo2"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_hybrid.HybridTowerConfig"
+  }
+}

vision_tower/configuration_hybrid.py

@@ -0,0 +1,48 @@
+import numpy as np
+import torch
+import torch.nn as nn
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
+import os
+import torch.nn.functional as F
+from transformers.modeling_utils import PreTrainedModel
+from transformers.configuration_utils import PretrainedConfig
+from transformers import AutoConfig
+from collections import OrderedDict
+
+
+class HybridTowerConfig(PretrainedConfig):
+    model_type = "hybrid_vision_tower"
+
+    def __init__(self, configs=None, **kwargs):
+        """
+        Initializes the HybridTowerConfig.
+        
+        Args:
+            configs (dict, optional): A dictionary where keys are component names and values are
+                                      instances of configurations that have a `to_dict()` method.
+            **kwargs: Additional keyword arguments that are passed to the superclass.
+        """
+        super().__init__(**kwargs)
+        self.configs = {}
+        
+        if configs is not None:
+            if not isinstance(configs, dict):
+                raise TypeError("configs must be a dictionary where keys are component names and values are configuration objects.")
+            
+            for component_name, config in configs.items():
+                if hasattr(config, 'to_dict'):
+                    self.configs[component_name] = config.to_dict()
+                else:
+                    raise TypeError(f"The configuration for '{component_name}' does not have a to_dict() method and cannot be serialized.")
+    
+    def to_dict(self):
+        """
+        Serializes this instance to a Python dictionary.
+
+        Returns:
+            dict: A dictionary containing all the keys and values of this configuration instance.
+        """
+        config_dict = super().to_dict()
+        config_dict['configs'] = self.configs
+        return config_dict
+

vision_tower/internvideo2/config.json

@@ -0,0 +1,54 @@
+{
+  "architectures": [
+    "PretrainInternVideo2"
+  ],
+  "attention_probs_dropout_prob": 0.0,
+  "attn_pool_num_heads": 16,
+  "checkpoint_num": 40,
+  "clip_embed_dim": 768,
+  "clip_input_resolution": 224,
+  "clip_norm_type": "l2",
+  "clip_return_layer": 6,
+  "clip_student_return_interval": 1,
+  "clip_teacher": null,
+  "clip_teacher_embed_dim": 3200,
+  "clip_teacher_final_dim": 768,
+  "clip_teacher_return_interval": 1,
+  "d_model": 1408,
+  "encoder_stride": 16,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.0,
+  "hidden_size": 768,
+  "image_mask_ratio": 0.5,
+  "image_mask_type": "random",
+  "image_size": 224,
+  "img_size": 224,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "keep_temporal": false,
+  "layer_norm_eps": 1e-12,
+  "model_type": "internvideo2",
+  "name": "pretrain_internvideo2_1b_patch14_224",
+  "num_attention_heads": 12,
+  "num_channels": 3,
+  "num_frames": 4,
+  "num_heads": 16,
+  "num_hidden_layers": 12,
+  "only_mask": true,
+  "patch_size": 14,
+  "qkv_bias": false,
+  "sep_image_video_pos_embed": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.0",
+  "tubelet_size": 1,
+  "use_checkpoint": true,
+  "use_flash_attn": false,
+  "use_fused_mlp": false,
+  "use_fused_rmsnorm": false,
+  "video_mask_ratio": 0.8,
+  "video_mask_type": "random",
+  "auto_map": {
+    "AutoConfig": "configuration_internvideo2.InternVideo2Config",
+    "AutoModel": "modeling_internvideo2.InternVideo2Model"
+  }
+}

vision_tower/internvideo2/configuration_internvideo2.py

@@ -0,0 +1,91 @@
+from transformers import PretrainedConfig
+
+
+class InternVideo2Config(PretrainedConfig):
+    model_type = "internvideo2"
+
+    def __init__(
+        self,
+        img_size=224,
+        patch_size=14,
+        tubelet_size=1,
+        num_frames=8,
+        d_model=1408,
+        num_heads=16,
+        depth=40,
+        mlp_ratio=48 / 11,
+        qkv_bias=False,
+        init_values=1e-5,
+        use_checkpoint=False,
+        checkpoint_num=0,
+        use_flash_attn=False,
+        use_fused_mlp=False,
+        use_fused_rmsnorm=False,
+        qk_normalization=True,
+        clip_embed_dim=1408,
+        attn_pool_num_heads=16,
+        clip_teacher_embed_dim=512,
+        clip_teacher_final_dim=512,
+        clip_student_return_interval=4,
+        clip_return_layer=3,
+        clip_norm_type="l2",
+        sep_image_video_pos_embed=False,
+        **kwargs,
+    ):
+        """
+        This is the configuration class to store the configuration of a `InternVideo2Model`.
+        It is used to instantiate a InternVideo2 model according to the specified arguments,
+        defining the model architecture.
+
+        Args:
+            img_size (int, optional): Input image size. Defaults to 224.
+            patch_size (int, optional): Size of each patch. Defaults to 14.
+            tubelet_size (int, optional): Temporal tubelet size. Defaults to 1.
+            num_frames (int, optional): Number of frames in the video input. Defaults to 8.
+            d_model (int, optional): Dimension of the model embeddings. Defaults to 1408.
+            num_heads (int, optional): Number of attention heads. Defaults to 16.
+            depth (int, optional): Number of transformer encoder layers. Defaults to 40.
+            mlp_ratio (float, optional): Ratio of MLP hidden dim to embedding dim. Defaults to 48/11.
+            qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Defaults to False.
+            init_values (float, optional): Initial values for layer scale. Defaults to 1e-5.
+            use_checkpoint (bool, optional): Whether to use gradient checkpointing. Defaults to False.
+            checkpoint_num (int, optional): Number of layers to apply checkpointing. Defaults to 0.
+            use_flash_attn (bool, optional): Whether to use FlashAttention. Defaults to False.
+            use_fused_mlp (bool, optional): Whether to use fused MLP. Defaults to False.
+            use_fused_rmsnorm (bool, optional): Whether to use fused RMSNorm. Defaults to False.
+            qk_normalization (bool, optional): Whether to apply QK normalization. Defaults to True.
+            clip_embed_dim (int, optional): Embedding dimension for CLIP. Defaults to 1408.
+            attn_pool_num_heads (int, optional): Number of heads for attention pooling. Defaults to 16.
+            clip_teacher_embed_dim (int, optional): Embedding dimension for CLIP teacher model. Defaults to 512.
+            clip_teacher_final_dim (int, optional): Final embedding dimension for CLIP teacher model. Defaults to 512.
+            clip_student_return_interval (int, optional): Interval for returning student layers. Defaults to 4.
+            clip_return_layer (int, optional): Number of layers to return for alignment. Defaults to 3.
+            clip_norm_type (str, optional): Normalization type for CLIP ('l2' or 'none'). Defaults to 'l2'.
+            sep_image_video_pos_embed (bool, optional): Whether to use separate position embeddings for image and video. Defaults to False.
+            **kwargs: Additional keyword arguments.
+        """
+        super().__init__(**kwargs)
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.tubelet_size = tubelet_size
+        self.num_frames = num_frames
+        self.d_model = d_model
+        self.num_heads = num_heads
+        self.depth = depth
+        self.mlp_ratio = mlp_ratio
+        self.qkv_bias = qkv_bias
+        self.init_values = init_values
+        self.use_checkpoint = use_checkpoint
+        self.checkpoint_num = checkpoint_num
+        self.use_flash_attn = use_flash_attn
+        self.use_fused_mlp = use_fused_mlp
+        self.use_fused_rmsnorm = use_fused_rmsnorm
+        self.qk_normalization = qk_normalization
+        self.clip_embed_dim = clip_embed_dim
+        self.attn_pool_num_heads = attn_pool_num_heads
+        self.clip_teacher_embed_dim = clip_teacher_embed_dim
+        self.clip_teacher_final_dim = clip_teacher_final_dim
+        self.clip_student_return_interval = clip_student_return_interval
+        self.clip_return_layer = clip_return_layer
+        self.clip_norm_type = clip_norm_type
+        self.sep_image_video_pos_embed = sep_image_video_pos_embed

vision_tower/internvideo2/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc98ff193aca2992ed2f56fea01c4d7be2e2c737cf9fcc5a73ef31663e728624
+size 2098289968

vision_tower/internvideo2/modeling_internvideo2.py

@@ -0,0 +1,934 @@
+# modeling_internvideo2.py
+
+import logging
+import math
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from transformers import PreTrainedModel
+from transformers.utils import logging as hf_logging
+
+from torch.utils.checkpoint import checkpoint  # Correct
+
+from functools import partial
+
+from .configuration_internvideo2 import InternVideo2Config  # Import the configuration
+
+try:
+    from einops import rearrange
+except ImportError:
+    raise ImportError("Please install einops to use this model.")
+
+try:
+    from timm.models.layers import DropPath, to_2tuple
+except ImportError:
+    raise ImportError("Please install timm to use this model.")
+
+logger = hf_logging.get_logger(__name__)
+
+# Position embedding functions
+def get_3d_sincos_pos_embed(embed_dim, grid_size, t_size, cls_token=False):
+    assert embed_dim % 4 == 0
+    embed_dim_spatial = embed_dim // 4 * 3
+    embed_dim_temporal = embed_dim // 4
+
+    # Spatial
+    grid_h = np.arange(grid_size, dtype=np.float32)
+    grid_w = np.arange(grid_size, dtype=np.float32)
+    grid = np.meshgrid(grid_w, grid_h)  # W first
+    grid = np.stack(grid, axis=0)
+
+    grid = grid.reshape([2, 1, grid_size, grid_size])
+    pos_embed_spatial = get_2d_sincos_pos_embed_from_grid(embed_dim_spatial, grid)
+
+    # Temporal
+    grid_t = np.arange(t_size, dtype=np.float32)
+    pos_embed_temporal = get_1d_sincos_pos_embed_from_grid(embed_dim_temporal, grid_t)
+
+    # Combine spatial and temporal embeddings
+    pos_embed_temporal = pos_embed_temporal[:, np.newaxis, :]
+    pos_embed_temporal = np.repeat(pos_embed_temporal, grid_size**2, axis=1)
+    pos_embed_spatial = pos_embed_spatial[np.newaxis, :, :]
+    pos_embed_spatial = np.repeat(pos_embed_spatial, t_size, axis=0)
+
+    pos_embed = np.concatenate([pos_embed_temporal, pos_embed_spatial], axis=-1)
+    pos_embed = pos_embed.reshape([-1, embed_dim])
+
+    if cls_token:
+        pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
+    return pos_embed
+
+def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
+    assert embed_dim % 2 == 0
+
+    emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0])
+    emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1])
+
+    emb = np.concatenate([emb_h, emb_w], axis=1)
+    return emb
+
+def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
+    assert embed_dim % 2 == 0
+    omega = np.arange(embed_dim // 2, dtype=np.float32)
+    omega /= embed_dim / 2.0
+    omega = 1.0 / (10000 ** omega)
+
+    pos = pos.reshape(-1)
+    out = np.einsum('m,d->md', pos, omega)
+
+    emb_sin = np.sin(out)
+    emb_cos = np.cos(out)
+
+    emb = np.concatenate([emb_sin, emb_cos], axis=1)
+    return emb
+
+# Define necessary classes: CrossAttention, AttentiveBlock, AttentionPoolingBlock, RMSNorm, LayerScale, Attention, Mlp, Block, PatchEmbed, Linear_Decoder
+
+
+class CrossAttention(nn.Module):
+    def __init__(
+        self,
+        dim,
+        num_heads=8,
+        qkv_bias=False,
+        qk_scale=None,
+        attn_drop=0.0,
+        proj_drop=0.0,
+        attn_head_dim=None,
+        out_dim=None,
+    ):
+        super().__init__()
+        if out_dim is None:
+            out_dim = dim
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        if attn_head_dim is not None:
+            head_dim = attn_head_dim
+        all_head_dim = head_dim * self.num_heads
+        self.scale = qk_scale or head_dim ** -0.5
+        assert all_head_dim == dim
+
+        self.q = nn.Linear(dim, all_head_dim, bias=False)
+        self.k = nn.Linear(dim, all_head_dim, bias=False)
+        self.v = nn.Linear(dim, all_head_dim, bias=False)
+
+        if qkv_bias:
+            self.q_bias = nn.Parameter(torch.zeros(all_head_dim))
+            self.k_bias = nn.Parameter(torch.zeros(all_head_dim))
+            self.v_bias = nn.Parameter(torch.zeros(all_head_dim))
+        else:
+            self.q_bias = None
+            self.k_bias = None
+            self.v_bias = None
+
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(all_head_dim, out_dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+    def forward(self, x, k=None, v=None):
+        B, N, C = x.shape
+        N_k = k.shape[1]
+        N_v = v.shape[1]
+
+        q_bias, k_bias, v_bias = None, None, None
+        if self.q_bias is not None:
+            q_bias = self.q_bias
+            k_bias = self.k_bias
+            v_bias = self.v_bias
+
+        q = F.linear(input=x, weight=self.q.weight, bias=q_bias)
+        q = (
+            q.reshape(B, N, 1, self.num_heads, -1)
+            .permute(2, 0, 3, 1, 4)
+            .squeeze(0)
+        )  # (B, N_head, N_q, dim)
+
+        k = F.linear(input=k, weight=self.k.weight, bias=k_bias)
+        k = (
+            k.reshape(B, N_k, 1, self.num_heads, -1)
+            .permute(2, 0, 3, 1, 4)
+            .squeeze(0)
+        )
+
+        v = F.linear(input=v, weight=self.v.weight, bias=v_bias)
+        v = (
+            v.reshape(B, N_v, 1, self.num_heads, -1)
+            .permute(2, 0, 3, 1, 4)
+            .squeeze(0)
+        )
+
+        q = q * self.scale
+        attn = q @ k.transpose(-2, -1)  # (B, N_head, N_q, N_k)
+
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B, N, -1)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+
+        return x
+
+
+class AttentiveBlock(nn.Module):
+    def __init__(
+        self,
+        dim,
+        num_heads,
+        qkv_bias=False,
+        qk_scale=None,
+        drop=0.0,
+        attn_drop=0.0,
+        drop_path=0.0,
+        norm_layer=nn.LayerNorm,
+        attn_head_dim=None,
+        out_dim=None,
+    ):
+        super().__init__()
+
+        self.norm1_q = norm_layer(dim)
+        self.norm1_k = norm_layer(dim)
+        self.norm1_v = norm_layer(dim)
+        self.cross_attn = CrossAttention(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            qk_scale=qk_scale,
+            attn_drop=attn_drop,
+            proj_drop=drop,
+            attn_head_dim=attn_head_dim,
+            out_dim=out_dim,
+        )
+
+        self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+    def forward(
+        self, x_q, x_kv, pos_q, pos_k, bool_masked_pos, rel_pos_bias=None
+    ):
+        x_q = self.norm1_q(x_q + pos_q)
+        x_k = self.norm1_k(x_kv + pos_k)
+        x_v = self.norm1_v(x_kv)
+        x = self.cross_attn(x_q, k=x_k, v=x_v)
+
+        return x
+
+
+class AttentionPoolingBlock(AttentiveBlock):
+    def forward(self, x):
+        x_q = x.mean(1, keepdim=True)
+        x_kv, pos_q, pos_k = x, 0, 0
+        x = super().forward(
+            x_q, x_kv, pos_q, pos_k, bool_masked_pos=None, rel_pos_bias=None
+        )
+        x = x.squeeze(1)
+        return x
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(
+            variance + self.variance_epsilon
+        )
+        return self.weight * hidden_states.to(input_dtype)
+
+
+class LayerScale(nn.Module):
+    def __init__(
+        self, dim, init_values=1e-5, inplace=False, force_fp32=False
+    ):
+        super().__init__()
+        self.inplace = inplace
+        self.weight = nn.Parameter(init_values * torch.ones(dim))
+        self.force_fp32 = force_fp32
+
+    @torch.cuda.amp.autocast(enabled=False)
+    def forward(self, x):
+        if self.force_fp32:
+            output_type = x.dtype
+            out = (
+                x.float().mul_(self.weight.float())
+                if self.inplace
+                else x.float() * self.weight.float()
+            )
+            return out.to(dtype=output_type)
+        else:
+            out = x.mul_(self.weight) if self.inplace else x * self.weight
+            return out
+
+
+class Attention(nn.Module):
+    def __init__(
+        self,
+        dim,
+        num_heads=8,
+        qkv_bias=False,
+        attn_drop=0.0,
+        proj_drop=0.0,
+        use_flash_attn=False,
+        causal=False,
+        norm_layer=nn.LayerNorm,
+        qk_normalization=False,
+        use_fused_rmsnorm=False,
+    ):
+        super().__init__()
+        assert (
+            dim % num_heads == 0
+        ), "dim should be divisible by num_heads"
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = head_dim ** -0.5
+        
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+        self.use_flash_attn = use_flash_attn
+        if use_flash_attn:
+            self.causal = causal
+            try:
+                from flash_attn.flash_attention import FlashAttention
+
+                self.inner_attn = FlashAttention(
+                    attention_dropout=attn_drop
+                )
+            except ImportError:
+                raise ImportError(
+                    "Please install flash_attn to use flash attention."
+                )
+
+        self.qk_normalization = qk_normalization
+        self.q_norm = norm_layer(dim) if qk_normalization else nn.Identity()
+        self.k_norm = norm_layer(dim) if qk_normalization else nn.Identity()
+        self.use_fused_rmsnorm = use_fused_rmsnorm
+
+    def _naive_attn(self, x):
+        B, N, C = x.shape
+        # print(x.shape, torch.cuda.memory_allocated(), torch.cuda.memory_allocated())
+        qkv = (
+            self.qkv(x)
+            .reshape(B, N, 3, self.num_heads, C // self.num_heads)
+            .permute(2, 0, 3, 1, 4)
+        )
+        q, k, v = qkv.unbind(
+            0
+        )  # make torchscript happy (cannot use tensor as tuple)
+
+        if self.qk_normalization:
+            B_, H_, N_, D_ = q.shape
+            q = (
+                self.q_norm(q.transpose(1, 2).flatten(-2, -1))
+                .view(B_, N_, H_, D_)
+                .transpose(1, 2)
+            )
+            k = (
+                self.k_norm(k.transpose(1, 2).flatten(-2, -1))
+                .view(B_, N_, H_, D_)
+                .transpose(1, 2)
+            )
+
+        attn = (q * self.scale) @ k.transpose(-2, -1)
+        # attn = attn - attn.max(-1)[0].unsqueeze(-1)  # in case of overflow for fp16
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+        # print(torch.cuda.memory_allocated(), torch.cuda.memory_allocated())
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+    def _flash_attn(
+        self, x, key_padding_mask=None, need_weights=False
+    ):
+        qkv = self.qkv(x)
+        qkv = rearrange(
+            qkv, "b s (three h d) -> b s three h d", three=3, h=self.num_heads
+        )
+
+        if self.qk_normalization:
+            q, k, v = qkv.unbind(2)
+            if self.use_fused_rmsnorm:
+                q = self.q_norm(q.flatten(-2, -1))[0].view(q.shape)
+                k = self.k_norm(k.flatten(-2, -1))[0].view(k.shape)
+            else:
+                q = self.q_norm(q.flatten(-2, -1)).view(q.shape)
+                k = self.k_norm(k.flatten(-2, -1)).view(k.shape)
+            qkv = torch.stack([q, k, v], dim=2)
+
+        context, _ = self.inner_attn(
+            qkv,
+            key_padding_mask=key_padding_mask,
+            need_weights=need_weights,
+            causal=self.causal,
+        )
+        outs = self.proj(rearrange(context, "b s h d -> b s (h d)"))
+        outs = self.proj_drop(outs)
+        return outs
+
+    def forward(self, x):
+        x = (
+            self._naive_attn(x)
+            if not self.use_flash_attn
+            else self._flash_attn(x)
+        )
+        return x
+
+
+class Mlp(nn.Module):
+    """MLP as used in Vision Transformer, MLP-Mixer and related networks"""
+
+    def __init__(
+        self,
+        in_features,
+        hidden_features=None,
+        out_features=None,
+        act_layer=nn.GELU,
+        bias=True,
+        drop=0.0,
+    ):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        bias = to_2tuple(bias)
+        drop_probs = to_2tuple(drop)
+
+        self.fc1 = nn.Linear(in_features, hidden_features, bias=bias[0])
+        self.act = act_layer()
+        self.drop1 = nn.Dropout(drop_probs[0])
+        self.fc2 = nn.Linear(hidden_features, out_features, bias=bias[1])
+        self.drop2 = nn.Dropout(drop_probs[1])
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop1(x)
+        x = self.fc2(x)
+        x = self.drop2(x)
+        return x
+
+
+class Block(nn.Module):
+    def __init__(
+        self,
+        dim,
+        num_heads,
+        mlp_ratio=4.0,
+        qkv_bias=False,
+        drop=0.0,
+        attn_drop=0.0,
+        init_values=None,
+        drop_path=0.0,
+        act_layer=nn.GELU,
+        norm_layer=nn.LayerNorm,
+        use_flash_attn=False,
+        use_fused_mlp=False,
+        fused_mlp_heuristic=1,
+        with_cp=False,
+        qk_normalization=False,
+        layerscale_no_force_fp32=False,
+        use_fused_rmsnorm=False,
+    ):
+        super().__init__()
+
+        self.norm1 = norm_layer(dim)
+        self.attn = Attention(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            attn_drop=attn_drop,
+            proj_drop=drop,
+            use_flash_attn=use_flash_attn,
+            causal=False,
+            norm_layer=norm_layer,
+            qk_normalization=qk_normalization,
+            use_fused_rmsnorm=use_fused_rmsnorm,
+        )
+        self.ls1 = (
+            LayerScale(
+                dim,
+                init_values=init_values,
+                force_fp32=(not layerscale_no_force_fp32),
+            )
+            if init_values
+            else nn.Identity()
+        )
+        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
+        self.drop_path1 = (
+            DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+        )
+
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        if use_fused_mlp:
+            try:
+                from flash_attn.modules.mlp import FusedMLP
+            except ImportError:
+                raise ImportError(
+                    "Please install flash_attn to use fused MLP."
+                )
+            self.mlp = FusedMLP(
+                in_features=dim,
+                hidden_features=mlp_hidden_dim,
+                heuristic=fused_mlp_heuristic,
+            )
+        else:
+            self.mlp = Mlp(
+                in_features=dim,
+                hidden_features=mlp_hidden_dim,
+                act_layer=act_layer,
+                drop=drop,
+            )
+        self.ls2 = (
+            LayerScale(
+                dim,
+                init_values=init_values,
+                force_fp32=(not layerscale_no_force_fp32),
+            )
+            if init_values
+            else nn.Identity()
+        )
+        self.drop_path2 = (
+            DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+        )
+
+        self.with_cp = with_cp
+        self.use_fused_rmsnorm = use_fused_rmsnorm
+
+    def forward(self, x, residual=None):
+        def _inner_forward(x, residual=None):
+            if self.use_fused_rmsnorm:
+                x, residual = self.norm1(x, residual)
+                x = self.drop_path1(self.ls1(self.attn(x)))
+                x, residual = self.norm2(x, residual)
+                x = self.drop_path2(self.ls2(self.mlp(x)))
+                return x, residual
+            else:
+                assert residual is None
+                x = x + self.drop_path1(
+                    self.ls1(self.attn(self.norm1(x)))
+                )
+                x = x + self.drop_path2(
+                    self.ls2(self.mlp(self.norm2(x)))
+                )
+                return x
+
+        if self.with_cp:
+            return checkpoint(_inner_forward, x, residual)
+        else:
+            return _inner_forward(x, residual=residual)
+
+
+class PatchEmbed(nn.Module):
+    """3D Image to Patch Embedding"""
+
+    def __init__(
+        self,
+        img_size=224,
+        patch_size=16,
+        in_chans=3,
+        embed_dim=768,
+        num_frames=8,
+        tubelet_size=1,
+        norm_layer=None,
+    ):
+        super().__init__()
+        img_size = to_2tuple(img_size)
+        patch_size = to_2tuple(patch_size)
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.grid_size = (
+            num_frames // tubelet_size,
+            img_size[0] // patch_size[0],
+            img_size[1] // patch_size[1],
+        )  # (T, H, W)
+        self.num_patches = (
+            self.grid_size[0] * self.grid_size[1] * self.grid_size[2]
+        )
+        self.num_img_patches = self.grid_size[1] * self.grid_size[2]
+
+        self.proj = nn.Conv3d(
+            in_channels=in_chans,
+            out_channels=embed_dim,
+            kernel_size=(tubelet_size, patch_size[0], patch_size[1]),
+            stride=(tubelet_size, patch_size[0], patch_size[1]),
+        )
+        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
+
+    def forward(self, x):
+        x = self.proj(x)
+        x = (
+            x.flatten(3)
+            .permute(0, 2, 3, 1)
+        )  # B x C x T x HW => B x T x HW x C
+        x = self.norm(x)
+        return x
+
+
+
+class Linear_Decoder(nn.Module):
+    def __init__(self, in_channels=1408, out_channels=3200, norm_layer=nn.LayerNorm, clip_norm_type='l2'):
+        super().__init__()
+        self.clip_norm_type = clip_norm_type
+        logger.info(f'Normalization Type: {clip_norm_type}')
+
+        self.head = nn.Linear(in_channels, out_channels)
+        self.norm = norm_layer(out_channels)
+
+    def forward(self, x):
+        x = self.norm(self.head(x))
+
+        if self.clip_norm_type == 'l2':
+            x = x / x.norm(dim=-1, keepdim=True)
+        elif self.clip_norm_type == 'none':
+            pass
+        else:
+            raise NotImplementedError
+
+        return x
+
+class InternVideo2Model(PreTrainedModel):
+    config_class = InternVideo2Config
+    base_model_prefix = "internvideo2"
+
+    def __init__(self, config: InternVideo2Config):
+        super().__init__(config)
+
+        in_chans = 3
+        drop_path_rate = 0.25
+        qk_normalization = config.qk_normalization
+        clip_embed_dim = config.clip_embed_dim
+        num_heads = config.num_heads
+        qkv_bias = config.qkv_bias
+        init_values = config.init_values
+        mlp_ratio = config.mlp_ratio
+        depth = config.depth
+        num_frames = config.num_frames
+        self.num_frames = num_frames
+        self.tubelet_size = config.tubelet_size
+        use_fused_mlp = config.use_fused_mlp
+        use_fused_rmsnorm = config.use_fused_rmsnorm
+        use_flash_attn = config.use_flash_attn
+        assert (
+            use_flash_attn
+            == use_fused_rmsnorm
+            == use_fused_mlp
+        ), "use_flash_attn, use_fused_rmsnorm and use_fused_mlp should be consistent"
+
+        self.use_flash_attn = use_flash_attn
+        embed_dim = config.d_model
+        self.embed_dim = embed_dim
+
+        self.depth = depth
+        self.clip_norm_type = config.clip_norm_type
+        self.return_index = []
+        for i in range(config.clip_return_layer):
+            self.return_index.append(
+                depth - int(i * config.clip_student_return_interval) - 1
+            )
+        logger.info(f"Normalization Type: {config.clip_norm_type}")
+        logger.info(f"Student Return Index: {self.return_index}")
+
+        if use_fused_rmsnorm:
+            try:
+                from flash_attn.ops.rms_norm import DropoutAddRMSNorm
+            except ImportError:
+                raise ImportError(
+                    "Please install flash_attn to use fused RMSNorm."
+                )
+            norm_layer_for_blocks = partial(
+                DropoutAddRMSNorm, eps=1e-6, prenorm=True
+            )
+        else:
+            norm_layer_for_blocks = partial(RMSNorm, eps=1e-6)
+        self.norm_layer_for_blocks = norm_layer_for_blocks
+        self.patch_embed = PatchEmbed(
+            config.img_size,
+            config.patch_size,
+            in_chans,
+            embed_dim,
+            num_frames=num_frames,
+            tubelet_size=self.tubelet_size,
+        )
+        num_patches = self.patch_embed.num_patches
+        num_img_patches = self.patch_embed.num_img_patches
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+
+        self.sep_pos_embed = False
+        self.sep_image_video_pos_embed = config.sep_image_video_pos_embed
+        if self.sep_pos_embed:
+            raise NotImplementedError
+        else:
+            if self.sep_image_video_pos_embed:
+                logger.info(
+                    "Use joint position embedding, for image and video we use different pos_embed."
+                )
+                self.pos_embed = nn.Parameter(
+                    torch.zeros(1, num_patches + 1, embed_dim)
+                )
+                self.img_pos_embed = nn.Parameter(
+                    torch.zeros(1, num_img_patches + 1, embed_dim)
+                )
+                # for CLIP decoder
+                self.clip_pos_embed = nn.Parameter(
+                    torch.zeros(1, num_patches + 1, embed_dim)
+                )
+                self.clip_img_pos_embed = nn.Parameter(
+                    torch.zeros(1, num_img_patches + 1, embed_dim)
+                )
+            else:
+                logger.info(
+                    "Use joint position embedding, for image and video we use same pos_embed."
+                )
+                self.pos_embed = nn.Parameter(
+                    torch.zeros(1, num_patches + 1, embed_dim)
+                )
+                self.clip_pos_embed = nn.Parameter(
+                    torch.zeros(1, num_patches + 1, embed_dim)
+                )
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]
+        # choose which layer to use checkpoint
+        with_cp_list = [False] * depth
+        if config.use_checkpoint:
+            for idx in range(depth):
+                if idx < config.checkpoint_num:
+                    with_cp_list[idx] = True
+        logger.info(f"Droppath rate: {dpr}")
+        logger.info(f"Checkpoint list: {with_cp_list}")
+
+        self.blocks = nn.ModuleList(
+            [
+                Block(
+                    embed_dim,
+                    num_heads,
+                    mlp_ratio,
+                    qkv_bias=qkv_bias,
+                    norm_layer=norm_layer_for_blocks,
+                    drop_path=dpr[i],
+                    init_values=init_values,
+                    attn_drop=0.0,
+                    use_flash_attn=use_flash_attn,
+                    use_fused_mlp=use_fused_mlp,
+                    fused_mlp_heuristic=1,
+                    with_cp=with_cp_list[i],
+                    qk_normalization=qk_normalization,
+                    layerscale_no_force_fp32=False,
+                    use_fused_rmsnorm=use_fused_rmsnorm,
+                )
+                for i in range(depth)
+            ]
+        )
+        self.clip_projector = AttentionPoolingBlock(
+            dim=embed_dim,
+            num_heads=config.attn_pool_num_heads,
+            qkv_bias=True,
+            qk_scale=None,
+            drop=0.0,
+            attn_drop=0.0,
+            norm_layer=partial(nn.LayerNorm, eps=1e-5),
+            out_dim=clip_embed_dim,
+        )
+
+        # CLIP decoder
+        self.clip_decoder = nn.ModuleList(
+            [
+                Linear_Decoder(
+                    in_channels=embed_dim,
+                    out_channels=config.clip_teacher_embed_dim,
+                    norm_layer=partial(nn.LayerNorm, eps=1e-5),
+                    clip_norm_type=config.clip_norm_type,
+                )
+                for _ in range(config.clip_return_layer)
+            ]
+        )
+        self.final_clip_decoder = nn.Identity()
+        if config.clip_teacher_final_dim > 0:
+            self.final_clip_decoder = Linear_Decoder(
+                in_channels=clip_embed_dim,
+                out_channels=config.clip_teacher_final_dim,
+                norm_layer=partial(nn.LayerNorm, eps=1e-5),
+                clip_norm_type=config.clip_norm_type,
+            )
+
+        # Removed initialization methods and code
+
+    @property
+    def dtype(self):
+        return self.patch_embed.proj.weight.dtype
+
+    def get_num_layers(self):
+        return len(self.blocks)
+
+    @torch.jit.ignore
+    def no_weight_decay(self):
+        return {
+            "pos_embed",
+            "pos_embed_spatial",
+            "pos_embed_temporal",
+            "pos_embed_cls",
+            "img_pos_embed",
+            "cls_token",
+            "clip_pos_embed",
+            "clip_pos_embed_spatial",
+            "clip_pos_embed_temporal",
+            "clip_pos_embed_cls",
+            "clip_img_pos_embed",
+        }
+
+    def forward(
+        self,
+        x,
+        mask=None,
+        use_image=False,
+        x_vis_return_idx=-1,
+        x_vis_only=False,
+    ):
+        x = self.patch_embed(x.type(self.dtype))
+        B, T, L, C = x.shape
+        x = x.view([B, T * L, C])
+
+        # Append cls token
+        cls_tokens = self.cls_token.expand(B, -1, -1)
+        x = torch.cat((cls_tokens, x), dim=1)
+
+        # Add positional embeddings
+        if self.sep_pos_embed:
+            raise NotImplementedError
+        else:
+            if use_image:
+                if self.sep_image_video_pos_embed:
+                    pos_embed = self.img_pos_embed
+                else:
+                    cls_pos_embed = self.pos_embed[:, 0:1, :]
+                    img_pos_embed = (
+                        self.pos_embed[:, 1:, :]
+                        .view(
+                            1,
+                            self.num_frames,
+                            self.patch_embed.num_patches // self.num_frames,
+                            self.embed_dim,
+                        )
+                        .mean(dim=1)
+                    )
+                    pos_embed = torch.cat(
+                        [cls_pos_embed, img_pos_embed], dim=1
+                    )
+            else:
+                pos_embed = self.pos_embed
+        x = x + pos_embed
+
+        # Mask tokens
+        if mask is not None:
+            x = x[~mask].reshape(B, -1, C)
+        else:
+            x = x.reshape(B, -1, C)
+
+        residual = None
+        x_clip = []
+        for idx, blk in enumerate(self.blocks):
+            if isinstance(x, tuple) and len(x) == 2:
+                x, residual = x
+            x = blk(x, residual=residual)
+            # Return intermediate features
+            if idx in self.return_index:
+                if isinstance(x, tuple) and len(x) == 2:
+                    tmp_x, tmp_residual = x
+                    if residual is not None:
+                        x_clip.append(tmp_x + tmp_residual)
+                else:
+                    x_clip.append(x)
+            if idx == (self.depth + x_vis_return_idx):
+                break
+
+        if isinstance(x, tuple) and len(x) == 2:
+            x, residual = x
+            if residual is not None:
+                x = x + residual
+
+        x_vis = x
+        if x_vis_only:
+            return x_vis
+
+        x_pool_vis = self.clip_projector(x_vis)
+        x_align = self.final_clip_decoder(x_pool_vis)
+
+        # Align CLIP
+        x_clip = torch.stack(x_clip)
+        K, B, _, C_CLIP = x_clip.shape
+        # Add positional embeddings
+        if self.sep_pos_embed:
+            raise NotImplementedError
+        else:
+            if use_image:
+                if self.sep_image_video_pos_embed:
+                    clip_pos_embed = self.clip_img_pos_embed
+                else:
+                    clip_cls_pos_embed = self.clip_pos_embed[:, 0:1, :]
+                    clip_img_pos_embed = (
+                        self.clip_pos_embed[:, 1:, :]
+                        .view(
+                            1,
+                            self.num_frames,
+                            self.patch_embed.num_patches // self.num_frames,
+                            self.embed_dim,
+                        )
+                        .mean(dim=1)
+                    )
+                    clip_pos_embed = torch.cat(
+                        [clip_cls_pos_embed, clip_img_pos_embed], dim=1
+                    )
+
+            else:
+                clip_pos_embed = self.clip_pos_embed
+
+        clip_pos_embed = clip_pos_embed.repeat(B, 1, 1)
+        if mask is not None:
+            x_clip = x_clip + clip_pos_embed[~mask].view(
+                B, -1, C_CLIP
+            ).unsqueeze(0).repeat(K, 1, 1, 1)
+        else:
+            x_clip = x_clip + clip_pos_embed.view(B, -1, C_CLIP).unsqueeze(
+                0
+            ).repeat(K, 1, 1, 1)
+
+        # CLIP decoder
+        x_clip_align = []
+        for idx, clip_decoder in enumerate(self.clip_decoder):
+            x_clip_align.append(clip_decoder(x_clip[idx]))
+        x_clip_align = torch.stack(x_clip_align)
+
+        return x_vis, x_pool_vis, x_clip_align, x_align
+
+
+    def load_pretrained_weights(self):
+        if self.config.pretrained is not None:
+            logger.info(f"Loading pretrained weights from {self.config.pretrained}")
+            state_dict = torch.load(self.config.pretrained, map_location='cpu')
+            
+            # Rename 'ls1.weight' to 'ls1.weight' and 'ls2.weight' to 'ls2.weight'
+            new_state_dict = {}
+            for key, value in state_dict.items():
+                if key.endswith('.ls1.weight'):
+                    new_key = key.replace('.ls1.weight', '.ls1.weight')
+                    new_state_dict[new_key] = value
+                elif key.endswith('.ls2.weight'):
+                    new_key = key.replace('.ls2.weight', '.ls2.weight')
+                    new_state_dict[new_key] = value
+                else:
+                    new_state_dict[key] = value
+            
+            # Load the adjusted state_dict
+            message = self.load_state_dict(new_state_dict, strict=False)
+            logger.info(message)
+        else:
+            logger.info("No pretrained weights provided.")

vision_tower/internvideo2/preprocessor_config.json

@@ -0,0 +1,30 @@
+{
+    "Resize": {
+        "size": 224,
+        "interpolation": "bilinear"
+    },
+    "CenterCrop": {
+        "size": [
+            224,
+            224
+        ]
+    },
+    "ClipToTensor": {
+        "channel_nb": 3,
+        "div_255": true,
+        "numpy": false
+    },
+    "Normalize": {
+        "mean": [
+            0.485,
+            0.456,
+            0.406
+        ],
+        "std": [
+            0.229,
+            0.224,
+            0.225
+        ]
+    },
+    "image_processor_type": "transforms"
+}

vision_tower/siglip-so400m-patch14-384/config.json

@@ -0,0 +1,19 @@
+{
+  "_name_or_path": "/opt/hpcaas/.mounts/fs-0663e2d3c38211883/home/orrzohar/Artemis/work_dirs/final_run/apollo-Qwen2.5-7B-Instruct-internvideo2-siglip-so400m-patch14-384-freeze-perciver_128_2-newprompt-ft/checkpoint-13300/vision_tower/siglip-so400m-patch14-384",
+  "architectures": [
+    "SiglipVisionModel"
+  ],
+  "attention_dropout": 0.0,
+  "hidden_act": "gelu_pytorch_tanh",
+  "hidden_size": 1152,
+  "image_size": 384,
+  "intermediate_size": 4304,
+  "layer_norm_eps": 1e-06,
+  "model_type": "siglip_vision_model",
+  "num_attention_heads": 16,
+  "num_channels": 3,
+  "num_hidden_layers": 27,
+  "patch_size": 14,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.0"
+}

vision_tower/siglip-so400m-patch14-384/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e9299e05ae6cc8366374648c1a8d202d57baa136e117c2203a23d135dbb0707
+size 856506120

vision_tower/siglip-so400m-patch14-384/preprocessor_config.json

@@ -0,0 +1,24 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "SiglipImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "processor_class": "SiglipProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 384,
+    "width": 384
+  }
+}