Upload InternVideo2_Classification_test

config.json

@@ -5,49 +5,50 @@
   "auto_map": {
     "AutoModel": "modeling_videochat2_classification.InternVideo2_Classification_test"
   },
-  "bridge": {
-    "extra_num_query_token": 64,
-    "name": "qformer",
-    "num_query_token": 32,
-    "qformer_attention_probs_dropout_prob": 0.1,
-    "qformer_drop_path_rate": 0.2,
-    "qformer_hidden_dropout_prob": 0.1
+  "model_config": {
+    "bridge": {
+      "extra_num_query_token": 64,
+      "name": "qformer",
+      "num_query_token": 32,
+      "qformer_attention_probs_dropout_prob": 0.1,
+      "qformer_drop_path_rate": 0.2,
+      "qformer_hidden_dropout_prob": 0.1
+    },
+    "freeze_bridge": false,
+    "freeze_llm": false,
+    "freeze_vision_encoder": false,
+    "llm": {
+      "lora_alpha": 32,
+      "lora_dropout": 0.1,
+      "lora_r": 16,
+      "name": "mistral_7b",
+      "pretrained_llm_path": "mistralai/Mistral-7B-Instruct-v0.3",
+      "use_lora": true
+    },
+    "loss": {
+      "use_vision_regression_loss": false
+    },
+    "pretrained_paths": {},
+    "use_flash_attention": true,
+    "vision_encoder": {
+      "checkpoint_num": 48,
+      "d_model": 1408,
+      "encoder_embed_dim": 1408,
+      "img_size": 224,
+      "name": "internvideo2-1B",
+      "num_frames": 8,
+      "origin_num_frames": 4,
+      "patch_size": 14,
+      "pretrained": null,
+      "sep_image_video_pos_embed": true,
+      "tubelet_size": 1,
+      "use_checkpoint": true,
+      "vit_add_ln": true,
+      "x_vis_only": true,
+      "x_vis_return_idx": -2
+    }
   },
-  "freeze_bridge": false,
-  "freeze_llm": false,
-  "freeze_vision_encoder": false,
-  "llm": {
-    "lora_alpha": 32,
-    "lora_dropout": 0.1,
-    "lora_r": 16,
-    "name": "mistral_7b",
-    "pretrained_llm_path": "mistralai/Mistral-7B-Instruct-v0.3",
-    "use_lora": true
-  },
-  "loss": {
-    "use_vision_regression_loss": false
-  },
-  "model_config": {},
   "model_type": "InternVideo2_Classification_test",
-  "pretrained_paths": {},
   "torch_dtype": "float32",
-  "transformers_version": "4.46.1",
-  "use_flash_attention": true,
-  "vision_encoder": {
-    "checkpoint_num": 48,
-    "d_model": 1408,
-    "encoder_embed_dim": 1408,
-    "img_size": 224,
-    "name": "internvideo2-1B",
-    "num_frames": 8,
-    "origin_num_frames": 4,
-    "patch_size": 14,
-    "pretrained": null,
-    "sep_image_video_pos_embed": true,
-    "tubelet_size": 1,
-    "use_checkpoint": true,
-    "vit_add_ln": true,
-    "x_vis_only": true,
-    "x_vis_return_idx": -2
-  }
+  "transformers_version": "4.46.1"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c2e92eec0623bf8e345a2310b4baff5fd2ecb0897a3b6eb94e5de89951a2de3c
-size 42488
+oid sha256:dad2c8637e76385541187f8578c922050cc7b9c704e774f1fdab50b623f3b517
+size 743024240

modeling_videochat2_classification.py

@@ -391,6 +391,9 @@ class InternVideo2_Classification_test(PreTrainedModel):
         super().__init__(config)
         self.conv1 = nn.Conv2d(1, 20, 5)
         self.conv2 = nn.Conv2d(20, 20, 5)
+        self.model_config = config.model_config
+        self.build_bridge()
+
 
     def forward(self, x):
         x = self.conv1(x)
@@ -399,6 +402,35 @@ class InternVideo2_Classification_test(PreTrainedModel):
     def test_lol(self, x):
         return x
 
+    def build_bridge(self):
+        
+        if 'qformer' in self.model_config.bridge.name.lower():
+            from transformers import BertTokenizer
+            self.qformer_tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", truncation_side="left")
+            self.qformer_tokenizer.add_special_tokens({"bos_token": "[DEC]"})
+            self.qformer_tokenizer.padding_side = "left"
+            if self.model_config.bridge.name == 'qformer':
+                self.qformer, self.query_tokens = build_qformer(
+                        self.model_config.bridge.num_query_token, self.model_config.vision_encoder.encoder_embed_dim,
+                        qformer_hidden_dropout_prob=self.model_config.bridge.qformer_hidden_dropout_prob,
+                        qformer_attention_probs_dropout_prob=self.model_config.bridge.qformer_attention_probs_dropout_prob,
+                        qformer_drop_path_rate=self.model_config.bridge.qformer_drop_path_rate,
+                )
+            self.qformer.resize_token_embeddings(len(self.qformer_tokenizer))
+            self.qformer.cls = None
+            self.extra_num_query_token = self.model_config.bridge.extra_num_query_token
+            if self.model_config.bridge.extra_num_query_token > 0:
+                logger.info(f"Add extra {self.model_config.bridge.extra_num_query_token} tokens in QFormer")
+                self.extra_query_tokens = nn.Parameter(
+                    torch.zeros(1, self.model_config.bridge.extra_num_query_token, self.query_tokens.shape[-1])
+                )
+            
+            self.freeze_bridge = self.model_config.get("freeze_bridge", False)
+            if self.freeze_bridge:
+                logger.info("freeze bridge")
+                freeze_module(self.qformer)
+                self.query_tokens.requires_grad = False
+
 if __name__ == "__main__":
 
     tokenizer =  AutoTokenizer.from_pretrained('OpenGVLab/InternVideo2-Chat-8B',trust_remote_code=True,use_fast=False)