Repository: huggingface/transformers
Pull request: Fix convert weights for InternVL

# What does this PR do?

Fix InternVL conversion script.

Fixes #38203

Changed files:
filename: src/transformers/models/internvl/__init__.py
patch: @@ -21,6 +21,7 @@
     from .configuration_internvl import *
     from .modeling_internvl import *
     from .processing_internvl import *
+    from .video_processing_internvl import *
 else:
     import sys
 

filename: src/transformers/models/internvl/convert_internvl_weights_to_hf.py
patch: @@ -28,6 +28,7 @@
     InternVLConfig,
     InternVLForConditionalGeneration,
     InternVLProcessor,
+    InternVLVideoProcessor,
     InternVLVisionConfig,
     LlamaConfig,
     Qwen2Config,
@@ -56,7 +57,7 @@
 # fmt: off
 ORIGINAL_TO_CONVERTED_KEY_MAPPING_VISION = {
     # Vision encoder mapping
-    r"vision_model":                                r"vision_tower",
+    r"vision_model":                                r"model.vision_tower",
     r"layers":                                      r"layer",
     r"class_embedding":                             r"cls_token",
     r"position_embedding":                          r"position_embeddings",
@@ -71,22 +72,28 @@
 }
 
 ORIGINAL_TO_CONVERTED_KEY_MAPPING_TEXT_LLAMA = {
-    # Vision encoder mapping
+    r"language_model.model.":                       r"model.language_model.",
     r"tok_embeddings":                              r"embed_tokens",
     r"attention.wo":                                r"self_attn.o_proj",
     r"feed_forward.w1":                             r"mlp.gate_proj",
     r"feed_forward.w2":                             r"mlp.down_proj",
     r"feed_forward.w3":                             r"mlp.up_proj",
     r"attention_norm":                              r"input_layernorm",
     r"ffn_norm":                                    r"post_attention_layernorm",
-    r"output":                                      r"lm_head",
+    r"language_model.output":                       r"lm_head",
+}
+
+ORIGINAL_TO_CONVERTED_KEY_MAPPING_TEXT_QWEN2 = {
+    # Vision encoder mapping
+    r"language_model.model.":                       r"model.language_model.",
+    r"language_model.lm_head":                       r"lm_head",
 }
 
 ORIGINAL_TO_CONVERTED_KEY_MAPPING_MULTI = {
     # Vision encoder mapping
-    r"mlp1.0":                                 r"multi_modal_projector.layer_norm",
-    r"mlp1.1":                                 r"multi_modal_projector.linear_1",
-    r"mlp1.3":                                 r"multi_modal_projector.linear_2",
+    r"mlp1.0":                                 r"model.multi_modal_projector.layer_norm",
+    r"mlp1.1":                                 r"model.multi_modal_projector.linear_1",
+    r"mlp1.3":                                 r"model.multi_modal_projector.linear_2",
 }
 
 
@@ -98,7 +105,7 @@
         "{% else %}"
             "{% for content in message['content'] %}"
                 "{% if content['type'] == 'image' %}"
-                    "{{ '<image>\n' }}"
+                    "{{ '<IMG_CONTEXT>\n' }}"
                 "{% elif content['type'] == 'video' %}"
                     "{{ '<video>\n' }}"
                 "{% elif content['type'] == 'text' %}"
@@ -134,6 +141,9 @@ def convert_old_keys_to_new_keys(state_dict_keys: Optional[dict] = None, path: O
         if LM_TYPE_CORRESPONDENCE[path] == "llama":
             for pattern, replacement in ORIGINAL_TO_CONVERTED_KEY_MAPPING_TEXT_LLAMA.items():
                 new_text = re.sub(pattern, replacement, new_text)
+        elif LM_TYPE_CORRESPONDENCE[path] == "qwen2":
+            for pattern, replacement in ORIGINAL_TO_CONVERTED_KEY_MAPPING_TEXT_QWEN2.items():
+                new_text = re.sub(pattern, replacement, new_text)
         output_dict.update(dict(zip(old_text_language.split("\n"), new_text.split("\n"))))
         old_text_multi = "\n".join(
             [
@@ -276,8 +286,14 @@ def write_model(
         model.push_to_hub(hub_dir, use_temp_dir=True)
 
     image_processor = GotOcr2ImageProcessorFast.from_pretrained(model_path)
+    video_processor = InternVLVideoProcessor.from_pretrained(model_path)
     tokenizer = AutoTokenizer.from_pretrained(model_path)
-    processor = InternVLProcessor(image_processor=image_processor, tokenizer=tokenizer, chat_template=chat_template)
+    processor = InternVLProcessor(
+        image_processor=image_processor,
+        video_processor=video_processor,
+        tokenizer=tokenizer,
+        chat_template=chat_template,
+    )
     processor.save_pretrained(model_path)
     if push_to_hub:
         processor.push_to_hub(hub_dir, use_temp_dir=True)
@@ -349,6 +365,7 @@ def write_tokenizer(
                 "start_image_token": "<img>",
                 "end_image_token": "</img>",
                 "context_image_token": "<IMG_CONTEXT>",
+                "video_token": "<video>",
             },
         )