Update

app.py

@@ -58,11 +58,11 @@ cfg = Config(parse_args())
 
 model_config = cfg.model_cfg
 model_cls = registry.get_model_class(model_config.arch)
-model = model_cls.from_config(model_config).to('cpu')
+model = model_cls.from_config(model_config).to('cuda:0')
 
 vis_processor_cfg = cfg.datasets_cfg.cc_sbu_align.vis_processor.train
 vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config(vis_processor_cfg)
-chat = Chat(model, vis_processor,device='cpu')
+chat = Chat(model, vis_processor)
 print('Initialization Finished')
 
 # ========================================

minigpt4/configs/models/minigpt4_vicuna0.yaml

@@ -5,7 +5,7 @@ model:
   image_size: 224
   drop_path_rate: 0
   use_grad_checkpoint: False
-  vit_precision: "fp32"
+  vit_precision: "fp16"
   freeze_vit: True
   freeze_qformer: True
 

minigpt4/models/base_model.py

@@ -178,14 +178,14 @@ class BaseModel(nn.Module):
         if low_resource:
             llama_model = PhiForCausalLM.from_pretrained(
                 llama_model_path,
-                torch_dtype=torch.float32,
+                torch_dtype=torch.float16,
                 load_in_8bit=True,
                 device_map={'': low_res_device}
             )
         else:
             llama_model = PhiForCausalLM.from_pretrained(
                 llama_model_path,
-                torch_dtype=torch.float32,
+                torch_dtype=torch.float16,
             )
 
         if lora_r > 0: