HuggingFaceM4

/

idefics2-8b

like 484

Hi
I use the following code and I receive the error "OutOfMemoryError: CUDA out of memory. Tried to allocate 112.00 MiB. GPU"

I have nvidia 3050. what can be the problem?

import torch

from transformers import AutoProcessor, AutoModelForVision2Seq
from transformers.image_utils import load_image
DEVICE = "cuda:0"
from transformers import BitsAndBytesConfig
from transformers import AwqConfig

quantization_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True,
bnb_4bit_compute_dtype=torch.float16
)
model = AutoModelForVision2Seq.from_pretrained(
"HuggingFaceM4/idefics2-8b",
torch_dtype=torch.float16,
quantization_config=quantization_config,
).to(DEVICE)