vl-7b-4bit

Runtime error

minhdang commited on Mar 21

Commit

efa875e

•

1 Parent(s): 5834081

Update inference.py

Files changed (1) hide show

inference.py CHANGED Viewed

@@ -36,7 +36,7 @@ from deepseek_vl.utils.conversation import Conversation
 from transformers import BitsAndBytesConfig
 from transformers import QuantoConfig
-quanto_config = QuantoConfig(weights="int8")
 def load_model(model_path):
     vl_chat_processor: VLChatProcessor = VLChatProcessor.from_pretrained(model_path)

 from transformers import BitsAndBytesConfig
 from transformers import QuantoConfig
+quanto_config = QuantoConfig(weights="int4")
 def load_model(model_path):
     vl_chat_processor: VLChatProcessor = VLChatProcessor.from_pretrained(model_path)