GPTQ model commit

config.json

@@ -18,23 +18,51 @@
     "num_key_value_heads": 8,
     "pad_token_id": 0,
     "pretraining_tp": 1,
-    "rms_norm_eps": 1e-05,
-    "rope_scaling": null,
-    "rope_theta": 10000.0,
-    "tie_word_embeddings": false,
-    "torch_dtype": "float16",
-    "transformers_version": "4.36.0.dev0",
-    "use_cache": true,
-    "vocab_size": 32000,
     "quantization_config": {
+        "batch_size": 1,
         "bits": 3,
-        "group_size": -1,
+        "block_name_to_quantize": null,
+        "cache_block_outputs": true,
         "damp_percent": 0.1,
         "desc_act": true,
+        "exllama_config": {
+            "version": 1
+        },
+        "group_size": -1,
+        "max_input_length": null,
+        "model_seqlen": null,
+        "module_name_preceding_first_block": null,
+        "modules_in_block_to_quantize": [
+            [
+                "self_attn.k_proj",
+                "self_attn.v_proj",
+                "self_attn.q_proj"
+            ],
+            [
+                "self_attn.o_proj"
+            ],
+            [
+                "mlp.up_proj",
+                "mlp.gate_proj"
+            ],
+            [
+                "mlp.down_proj"
+            ]
+        ],
+        "pad_token_id": null,
+        "quant_method": "gptq",
         "sym": true,
+        "tokenizer": null,
         "true_sequential": true,
-        "model_name_or_path": null,
-        "model_file_base_name": "model",
-        "quant_method": "gptq"
-    }
+        "use_cuda_fp16": false,
+        "use_exllama": true
+    },
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": null,
+    "rope_theta": 10000.0,
+    "tie_word_embeddings": false,
+    "torch_dtype": "float16",
+    "transformers_version": "4.37.0.dev0",
+    "use_cache": true,
+    "vocab_size": 32000
 }

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
+  "transformers_version": "4.37.0.dev0"
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eff559e3dfd34ae96b7a46e9fbc518ea77280bff4720612e8e911edbef56c392
-size 39638178920
+oid sha256:505c05be240795ac2ba70a4abec3489cf0e694913e918ea0478d8fd4a30083c3
+size 39638178800

quantize_config.json

@@ -1,10 +1,8 @@
 {
-  "bits": 3,
-  "group_size": -1,
-  "damp_percent": 0.1,
-  "desc_act": true,
-  "sym": true,
-  "true_sequential": true,
-  "model_name_or_path": null,
-  "model_file_base_name": "model"
+    "bits": 3,
+    "group_size": -1,
+    "damp_percent": 0.1,
+    "desc_act": true,
+    "sym": true,
+    "true_sequential": true
 }