Update inference.py

inference.py

@@ -34,15 +34,18 @@ from deepseek_vl.models import MultiModalityCausalLM, VLChatProcessor
 from deepseek_vl.utils.conversation import Conversation
 
 from transformers import BitsAndBytesConfig
+from transformers import QuantoConfig
 
+quanto_config = QuantoConfig(weights="int8")
 
 def load_model(model_path):
     vl_chat_processor: VLChatProcessor = VLChatProcessor.from_pretrained(model_path)
     tokenizer = vl_chat_processor.tokenizer
     vl_gpt: MultiModalityCausalLM = AutoModelForCausalLM.from_pretrained(
-        model_path, trust_remote_code=True, load_in_8bit=True,low_cpu_mem_usage=True,device_map="auto"
+        model_path, trust_remote_code=True, quantization_config = quanto_config,low_cpu_mem_usage=True
     )
-    vl_gpt = vl_gpt.eval()
+    
+    vl_gpt = vl_gpt.to(torch.bfloat16).cuda().eval()
     return tokenizer, vl_gpt, vl_chat_processor