qtnx commited on
Commit
d806c78
1 Parent(s): 8904048

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +13 -2
README.md CHANGED
@@ -23,11 +23,22 @@ pip install torch transformers pillow
23
  import torch
24
  from PIL import Image
25
  from transformers import AutoModelForCausalLM, AutoTokenizer
 
 
 
 
 
 
 
 
26
 
27
  model_id = "qresearch/llama-3-vision-alpha-hf"
28
  model = AutoModelForCausalLM.from_pretrained(
29
- model_id, trust_remote_code=True, torch_dtype=torch.float16
30
- ).to("cuda")
 
 
 
31
 
32
  tokenizer = AutoTokenizer.from_pretrained(
33
  model_id,
 
23
  import torch
24
  from PIL import Image
25
  from transformers import AutoModelForCausalLM, AutoTokenizer
26
+ from transformers import BitsAndBytesConfig
27
+
28
+
29
+ bnb_cfg = BitsAndBytesConfig(
30
+ load_in_4bit=True,
31
+ bnb_4bit_compute_dtype=torch.float16,
32
+ llm_int8_skip_modules=["mm_projector", "vision_model"],
33
+ )
34
 
35
  model_id = "qresearch/llama-3-vision-alpha-hf"
36
  model = AutoModelForCausalLM.from_pretrained(
37
+ model_name,
38
+ trust_remote_code=True,
39
+ torch_dtype=torch.float16,
40
+ quantization_config=bnb_cfg,
41
+ )
42
 
43
  tokenizer = AutoTokenizer.from_pretrained(
44
  model_id,