Spaces:

pyakhurel
/

test-finetuned

Runtime error

pyakhurel commited on Oct 13, 2023

Commit

6721da1

1 Parent(s): 1f94b56

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -10,7 +10,7 @@ model_name = "bn22/Mistral-7B-Instruct-v0.1-sharded"
 device = "cuda"
 bnb_config = transformers.BitsAndBytesConfig(
-    load_in_4bit=True,
     bnb_4bit_use_double_quant=True,
     bnb_4bit_quant_type="nf4",
     bnb_4bit_compute_dtype=torch.bfloat16

 device = "cuda"
 bnb_config = transformers.BitsAndBytesConfig(
+    load_in_8bit=True
     bnb_4bit_use_double_quant=True,
     bnb_4bit_quant_type="nf4",
     bnb_4bit_compute_dtype=torch.bfloat16