marcsun13
/

opt-125m-awq

Text Generation

Inference Endpoints

text-generation-inference

4-bit precision

Model card Files Files and versions Community

Update config.json

#2

by ybelkada - opened Oct 30, 2023

base: refs/heads/main

←

from: refs/pr/2

Discussion Files changed

Files changed (1) hide show

config.json +2 -2

config.json CHANGED Viewed

@@ -25,10 +25,10 @@
   "prefix": "</s>",
   "quantization_config": {
     "backend": "autoawq",
-    "q_group_size": 128,
     "quant_method": "awq",
     "version": "GEMM",
-    "w_bit": 4,
     "zero_point": true
   },
   "torch_dtype": "float16",

   "prefix": "</s>",
   "quantization_config": {
     "backend": "autoawq",
+    "group_size": 128,
     "quant_method": "awq",
     "version": "GEMM",
+    "bits": 4,
     "zero_point": true
   },
   "torch_dtype": "float16",