fix proj moel

bubogpt/models/mm_gpt4.py

@@ -276,7 +276,7 @@ class MMGPT4(BaseModel):
         with_bind_head = cfg.get("with_bind_head", False)
         freeze_llm = cfg.get("freeze_llm", True)
         use_blip_vision = cfg.get("use_blip_vision", False)
-        proj_model = cfg.get("proj_model", "checkpoints/prerained_minigpt4_7b.pth")
+        proj_model = cfg.get("proj_model", "")
 
         model = cls(
             joiner_cfg=joiner_cfg,

imagebind/models/image_bind.py

@@ -656,8 +656,9 @@ def replace_joiner_vision(joiner, q_former_model, proj_model):
 
     joiner.modality_qformers[ModalityType.VISION].load_Qformer(q_former_model)
 
-    state_dict = torch.load(proj_model, map_location="cpu")["model"]
-    params = type(state_dict)()
-    params["fc.weight"] = state_dict["llama_proj.weight"]
-    params["fc.bias"] = state_dict["llama_proj.bias"]
-    joiner.modality_post_projectors[ModalityType.VISION].load_state_dict(params, strict=False)
+    if proj_model:
+        state_dict = torch.load(proj_model, map_location="cpu")["model"]
+        params = type(state_dict)()
+        params["fc.weight"] = state_dict["llama_proj.weight"]
+        params["fc.bias"] = state_dict["llama_proj.bias"]
+        joiner.modality_post_projectors[ModalityType.VISION].load_state_dict(params, strict=False)