Update README.md

README.md

@@ -42,20 +42,29 @@ dtype: bfloat16
 <summary>Setup</summary>
 
 ```python
-!pip install -qU transformers accelerate
+!pip install -qU transformers accelerate bitsandbytes
 
-from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import (
+    AutoTokenizer, 
+    AutoModelForCausalLM, 
+    BitsAndBytesConfig
+)
 import torch
 
 model = "dfurman/HermesBagel-34B-v0.1"
+nf4_config = BitsAndBytesConfig(
+   load_in_4bit=True,
+   bnb_4bit_quant_type="nf4",
+   bnb_4bit_use_double_quant=True,
+   bnb_4bit_compute_dtype=torch.bfloat16
+)
 
 tokenizer = AutoTokenizer.from_pretrained(model)
-
 model = AutoModelForCausalLM.from_pretrained(
     model,
     torch_dtype=torch.bfloat16,
     device_map="auto",
-    trust_remote_code=True,
+    quantization_config=nf4_config,
 )
 ```