minhdang commited on
Commit
c2bc52a
1 Parent(s): 8ef6bba

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -1
app.py CHANGED
@@ -9,7 +9,9 @@ from transformers import BitsAndBytesConfig
9
 
10
  nf4_config = BitsAndBytesConfig(
11
  load_in_4bit=True,
12
- bnb_4bit_use_double_quant=True,
 
 
13
  )
14
  MAX_MAX_NEW_TOKENS = 2048
15
  DEFAULT_MAX_NEW_TOKENS = 1024
 
9
 
10
  nf4_config = BitsAndBytesConfig(
11
  load_in_4bit=True,
12
+ bnb_4bit_use_double_quant=True,
13
+ bnb_4bit_quant_type="nf4",
14
+ bnb_4bit_compute_dtype=torch.bfloat16
15
  )
16
  MAX_MAX_NEW_TOKENS = 2048
17
  DEFAULT_MAX_NEW_TOKENS = 1024