TheBloke commited on
Commit
30e5e6c
1 Parent(s): 2263dd9

Update for Transformers AWQ support

Browse files
Files changed (1) hide show
  1. config.json +9 -2
config.json CHANGED
@@ -27,5 +27,12 @@
27
  "torch_dtype": "float16",
28
  "transformers_version": "4.35.0",
29
  "use_cache": true,
30
- "vocab_size": 32256
31
- }
 
 
 
 
 
 
 
 
27
  "torch_dtype": "float16",
28
  "transformers_version": "4.35.0",
29
  "use_cache": true,
30
+ "vocab_size": 32256,
31
+ "quantization_config": {
32
+ "quant_method": "awq",
33
+ "zero_point": true,
34
+ "group_size": 128,
35
+ "bits": 4,
36
+ "version": "gemm"
37
+ }
38
+ }