GPTQ model commit

config.json

@@ -18,25 +18,20 @@
     "num_key_value_heads": 8,
     "pad_token_id": 0,
     "pretraining_tp": 1,
-    "rms_norm_eps": 1e-05,
-    "rope_scaling": null,
-    "rope_theta": 10000.0,
-    "tie_word_embeddings": false,
-    "torch_dtype": "float16",
-    "transformers_version": "4.37.0.dev0",
-    "use_cache": true,
-    "vocab_size": 32002,
     "quantization_config": {
+        "batch_size": 1,
         "bits": 3,
-        "group_size": 128,
+        "block_name_to_quantize": null,
+        "cache_block_outputs": true,
         "damp_percent": 0.1,
         "desc_act": true,
-        "static_groups": false,
-        "sym": true,
-        "true_sequential": true,
-        "model_name_or_path": null,
-        "model_file_base_name": "model",
-        "quant_method": "gptq",
+        "exllama_config": {
+            "version": 1
+        },
+        "group_size": 128,
+        "max_input_length": null,
+        "model_seqlen": null,
+        "module_name_preceding_first_block": null,
         "modules_in_block_to_quantize": [
             [
                 "self_attn.k_proj",
@@ -53,6 +48,21 @@
             [
                 "mlp.down_proj"
             ]
-        ]
-    }
+        ],
+        "pad_token_id": null,
+        "quant_method": "gptq",
+        "sym": true,
+        "tokenizer": null,
+        "true_sequential": true,
+        "use_cuda_fp16": false,
+        "use_exllama": true
+    },
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": null,
+    "rope_theta": 10000.0,
+    "tie_word_embeddings": false,
+    "torch_dtype": "float16",
+    "transformers_version": "4.37.0.dev0",
+    "use_cache": true,
+    "vocab_size": 32002
 }

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 32000,
+  "pad_token_id": 0,
+  "transformers_version": "4.37.0.dev0"
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:232fda050d110068c0c3fea2c3daceab76b77cd5453958f148e304774142b076
-size 48264579392
+oid sha256:f82121d63701a147db5bbb1e10f08dcdbc2344ccfc955e151faecba27828a713
+size 48264579272

quantize_config.json

@@ -1,11 +1,8 @@
 {
-  "bits": 3,
-  "group_size": 128,
-  "damp_percent": 0.1,
-  "desc_act": true,
-  "static_groups": false,
-  "sym": true,
-  "true_sequential": true,
-  "model_name_or_path": null,
-  "model_file_base_name": "model"
+    "bits": 3,
+    "group_size": 128,
+    "damp_percent": 0.1,
+    "desc_act": true,
+    "sym": true,
+    "true_sequential": true
 }